apache_log_report 1.1.8 → 1.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d957c3ba44653ff3812aa0ad5f912edae8691f32c900cee509a005748319de81
-  data.tar.gz: 792424f5272630630ea896599aa0f814f72dc1ff599cb0f238327d17a603044f
+  metadata.gz: ffde49bcae6f56e31b5ef7cccd88d5391127ee330264c188132f9ccbec94f421
+  data.tar.gz: 04c799c68ab676b176314b7ebe68cd3514faa48f853b9793e76d505d71342ae9
 SHA512:
-  metadata.gz: 0b649cf5abc5e186372f853f2f127ea3afd6eb514f13ee2a006bf9310c4acc2e92f8fdd22414609fdd13a1197c377b3ccc9fde73d6cf95d6c9930056e90409aa
-  data.tar.gz: 7085f94df8b7029fd1fb8c011a9b187e0490f86501abacf8d205c608eb554a96abe70bc792ead20e3d5fc4f8b27572209f5a28f4b2ea2d0bb4c74498ef93e768
+  metadata.gz: 1b5010ca3fa37d6dceb8ab48d0fc5f256713db19f63366602f7200219c9ec51656160e2e50901709179fac67633f5703eb137432c5cd289bc080f14dc675aebf
+  data.tar.gz: 377a4a516fe144f6167c83850c0037cb9d0c216090ad197b2ca22b94d6ab156d1c696ceb31ab5c0be6f06227e7ef62dfd8a23667b26a9919e56eff431c9c977c

data/Gemfile.lock

@@ -1,9 +1,10 @@
 PATH
   remote: .
   specs:
-    apache_log_report (1.1.7)
+    apache_log_report (1.1.8)
       apache_log-parser
       browser
+      ipaddr
       sqlite3
       terminal-table
 
@@ -12,11 +13,12 @@ GEM
   specs:
     apache_log-parser (3.1.2)
     browser (5.3.1)
+    ipaddr (1.2.3)
     rake (12.3.3)
     sqlite3 (1.4.2)
-    terminal-table (2.0.0)
-      unicode-display_width (~> 1.1, >= 1.1.1)
-    unicode-display_width (1.7.0)
+    terminal-table (3.0.2)
+      unicode-display_width (>= 1.1.1, < 3)
+    unicode-display_width (2.1.0)
 
 PLATFORMS
   ruby

data/README.org

@@ -2,115 +2,6 @@
 #+AUTHOR: Adolfo Villafiorita
 #+STARTUP: showall
 
-* Introduction
+This Gem has been superseded by [[https://rubygems.org/gems/log_sense][Log Sense]], which produces better
+outputs and parses also rails logs.
 
-ApacheLogReport generates reports and statistics from Apache web logs
-in the =combined= format.  Written in Ruby, it runs from the command
-line, it is fast, and it can be installed on any system which supports
-Ruby.
-
-ApacheLogReport moves along the lines of tools such as [[https://goaccess.io/][GoAccess]]
-and [[https://umami.is/][Umami]], focusing on privacy and data-ownership: the data
-generated by ApacheLogReport is stored on your computer and owned by
-you (like it should be).
-
-ApacheLogReport is also inspired by static websites generators:
-statistics are generated from the command line and accessed as static
-HTML files.  By generating static resources, ApacheLogReport
-significantly reduces the attack surface of your webserver and
-installation headaches.
-
-We have, for instance, a cron job running on our servers, generating
-statistics at night.  The generated files are then made available on a
-private area on the web.
-
-Statistics are generated from Apache log formats in the =combined=
-format.  Reports are tailored, but not limited, to web servers serving
-static websites.  No need to install Java Script code on your
-websites, no cookies installed, no user tracking.
-
-ApacheLogReport reports the following data:
-
-- Visitors, hits, unique visitors, bandwidth used
-- Most accessed HTML pages
-- Most accessed resources  
-- Response statuses
-- referers
-- OS, browsers, and devices
-- Streaks: resources accessed by a given IP over time
-- Potential attacks: access to resources which are not meant to be
-  served by a web server serving static websites
- 
-Filters from the command line allow to analyze specific periods and
-distinguish traffic generated by self polls and crawlers.
-
-ApacheLogReport generates HTML and SQLite outputs.  Moreover, it can
-also generate reports in Org Mode format, which can be then processed
-to various formats (including LaTeX and HTML).
-
-* Installation
-
-  #+begin_src bash
-  gem install apache_log_report
-  #+end_src
-
-* Usage
-
-  #+begin_src bash :results raw output :wrap example
-  apache_log_report --help
-  #+end_src
-
-  #+RESULTS:
-  #+begin_example
-  Usage: apache_log_report [options] [logfile]
-      -l, --limit=N                    Number of entries to show (defaults to 30)
-      -b, --begin=DATE                 Consider entries after or on DATE
-      -e, --end=DATE                   Consider entries before or on DATE
-      -i, --ignore-crawlers            Ignore crawlers
-      -p, --ignore-selfpoll            Ignore apaches self poll entries (from ::1)
-          --only-crawlers              Perform analysis on crawlers only
-      -u, --prefix=PREFIX              Prefix to add to all plots (used to run multiple analyses in the same dir)
-      -w, --suffix=SUFFIX              Suffix to add to all plots (used to run multiple analyses in the same dir)
-      -c, --code-export=WHAT           Control :export directive in Org Mode code blocks (code, results, *both*, none)
-      -f, --format=FORMAT              Output format: html, org, sqlite. Defaults to org mode
-      -v, --version                    Prints version information
-      -h, --help                       Prints this help
-  This is version 1.1.6
-  #+end_example
-
-* Change Log
-
-See the [[file:CHANGELOG.org][CHANGELOG]] file.
-
-* Todo
-
-** TODO Referers should only include the hostname?
-** TODO Graphs in HTML output
-** TODO Countries/IP Lookup
-** TODO Light and Dark Theme
-** TODO Embed CSS
-** TODO Declare datatypes in table outputs, so that we can format data
-
-* Compatibility
-
-ApacheLogReport should run on any system on which Ruby runs.
-
-Concerning the outputs:
-
-- The HTML report uses [[https://picturepan2.github.io/spectre/][Spectre.css]] and (will use) [[https://vega.github.io/vega-lite/][Vega Light]], which
-  are downloaded from a CDN
-- Org Mode plots data using [[http://gnuplot.info/][Gnuplot]]
-
-* Author and Contributors
-
-[[http://ict4g.net/adolfo][Adolfo Villafiorita]].
-
-* Known Bugs
-
-Some known bugs and an unknown number of unknown bugs.
-
-(See the open issues for the known bugs.)
-
-* License
-
-Distributed under the terms of the [[http://opensource.org/licenses/MIT][MIT License]].

data/Rakefile

@@ -1,2 +1,15 @@
 require "bundler/gem_tasks"
 task :default => :spec
+
+require 'rake/testtask'
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+end
+
+require_relative './lib/apache_log_report/ip_locator.rb'
+
+desc "Convert Geolocation DB to sqlite"
+task :dbip_to_sqlite3, [:filename] do |tasks, args|
+  filename = args[:filename]
+  ApacheLogReport::IpLocator::dbip_to_sqlite filename
+end

data/apache_log_report.gemspec

@@ -29,6 +29,18 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "apache_log-parser"
   spec.add_dependency "browser"
+  spec.add_dependency "ipaddr"
+  spec.add_dependency "iso_country_codes"
   spec.add_dependency "sqlite3"
   spec.add_dependency "terminal-table"
+
+  spec.add_development_dependency "minitest"
+
+  spec.post_install_message = <<-MESSAGE
+  !    The 'apache_log_report' gem has been deprecated and has been replaced 'log_sense'.
+  !    See: https://rubygems.org/gems/log_sense
+  !    And: https://www.ict4g.net/gitea/adolfo/log_sense
+  MESSAGE
+
+
 end

data/exe/apache_log_report

@@ -31,6 +31,7 @@ if @options[:output_format]  == "sqlite"
   b.finish
 else
   @data = ApacheLogReport::DataCruncher.crunch @db, @options
+  @data = ApacheLogReport::DataCruncher.geolocate @data
 
   @ended_at = Time.now
   @duration = @ended_at - @started_at

data/lib/apache_log_report/data_cruncher.rb

@@ -1,7 +1,5 @@
-
 module ApacheLogReport
   module DataCruncher
-
     #
     # take a sqlite3 database and analyze data
     #
@@ -103,12 +101,13 @@ module ApacheLogReport
 
       @browsers = db.execute "SELECT browser, count(browser), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by browser order by count(browser) desc"
       @platforms = db.execute "SELECT platform, count(platform), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by platform order by count(platform) desc"
-      @ips = db.execute "SELECT ip, count(ip), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by ip order by count(ip) desc limit #{options[:limit]}"
       @referers = db.execute "SELECT referer, count(referer), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by referer order by count(referer) desc limit #{options[:limit]}"
       
-      @streaks = db.execute "SELECT ip, substr(datetime, 1, 10), path from LogLine order by ip, datetime"
+      @ips = db.execute "SELECT ip, count(ip), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by ip order by count(ip) desc limit #{options[:limit]}"
 
+      @streaks = db.execute "SELECT ip, substr(datetime, 1, 10), path from LogLine order by ip, datetime"
       data = {}
+
       self.instance_variables.each do |variable|
         var_as_symbol = variable.to_s[1..-1].to_sym
         data[var_as_symbol] = eval(variable.to_s)
@@ -116,6 +115,16 @@ module ApacheLogReport
       data
     end
 
+    # add country code to data[:ips]
+    def self.geolocate data
+      @location_db = IpLocator::load_db
+      data[:ips].each do |ip|
+        country_code = IpLocator::locate_ip ip[0], @location_db
+        ip << country_code
+      end
+      data
+    end
+
     private
 
     def self.date_intersect date1, date2, method
@@ -127,8 +136,6 @@ module ApacheLogReport
         date2
       end
     end
-
-
   end
 end
 

data/lib/apache_log_report/ip_locator.rb

@@ -0,0 +1,42 @@
+require 'csv'
+require 'sqlite3'
+require 'ipaddr'
+require 'iso_country_codes'
+
+module ApacheLogReport
+  module IpLocator
+    DB_FILE = "ip_locations/dbip-country-lite.sqlite3"
+
+    def self.dbip_to_sqlite db_location
+      db = SQLite3::Database.new ":memory:"
+      db.execute "CREATE TABLE ip_location (
+        from_ip_n INTEGER,
+        from_ip TEXT,
+        to_ip TEXT,
+        country_code TEXT
+      )"
+
+      ins = db.prepare "INSERT INTO ip_location(from_ip_n, from_ip, to_ip, country_code) values (?, ?, ?, ?)"
+      CSV.foreach(db_location) do |row|
+        ip = IPAddr.new row[0]
+        ins.execute(ip.to_i, row[0], row[1], row[2])
+      end
+
+      # persist to file
+      ddb = SQLite3::Database.new(DB_FILE)
+      b = SQLite3::Backup.new(ddb, 'main', db, 'main')
+      b.step(-1) #=> DONE
+      b.finish
+    end
+
+    def self.load_db
+      SQLite3::Database.new DB_FILE
+    end
+
+    def self.locate_ip ip, db
+      ip_n = IPAddr.new(ip).to_i
+      res = db.execute "SELECT * FROM ip_location where from_ip_n <= #{ip_n} order by from_ip_n desc limit 1"
+      IsoCountryCodes.find(res[0][3]).name
+    end
+  end
+end

data/lib/apache_log_report/templates/template.html.erb

@@ -32,6 +32,8 @@
                 "Platforms",
                 "Referers",
                 "IPs",
+                "Geolocation",
+                "Streaks",
                 "Command Invocation",
                 "Performance"
               ].each do |item| %>
@@ -142,7 +144,7 @@
           { title: "Browsers", header: ["Browser", "Hits", "Visitors", "Size"], rows: data[:browsers] },
           { title: "Platforms", header: ["Platform", "Hits", "Visitors", "Size"], rows: data[:platforms] },
           { title: "Referers", header: ["Referers", "Hits", "Visitors", "Size"], rows: data[:referers], col: "col-12" },
-          { title: "IPs", header: ["IPs", "Hits", "Visitors", "Size"], rows: data[:ips] },
+          { title: "IPs", header: ["IPs", "Hits", "Visitors", "Size", "Country"], rows: data[:ips] },
           { },
         ]
         %>
@@ -161,6 +163,31 @@
           <% end %>
         </div>
 
+        <article>
+          <h2 id="geolocation">Geolocation</h2>
+          <table class="table">
+            <thead>
+              <tr>
+                <th>Country Code</th>
+                <th>Total Hits</th>
+                <th>Total Visitors</th>
+                <th>IPs</th>
+              </tr>              
+            </thead>
+            <tbody>
+              <%# IP, Hits, Visitors Size, Country%>
+              <% data[:ips].group_by { |x| x[4] }.each do |k, v| %>
+                <tr>
+                  <td><%= k %></td>
+                  <td><%= v.map { |x| x[1] }.inject(&:+) %></td>
+                  <td><%= v.map { |x| x[2] }.inject(&:+) %></td>
+                  <td><%= v.map { |x| x[0] }.join(", ") %></td>
+                </tr>
+              <% end %>
+            </tbody>
+          </table>
+        </article>
+
         <article>
           <h2 id="streaks">Streaks</h2>
 

data/lib/apache_log_report/version.rb

@@ -1,3 +1,3 @@
 module ApacheLogReport
-  VERSION = "1.1.8"
+  VERSION = "1.1.9"
 end

data/lib/apache_log_report.rb

@@ -2,4 +2,5 @@ require 'apache_log_report/version'
 require 'apache_log_report/options_parser'
 require 'apache_log_report/log_parser'
 require 'apache_log_report/data_cruncher'
+require 'apache_log_report/ip_locator'
 require 'apache_log_report/emitter'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: apache_log_report
 version: !ruby/object:Gem::Version
-  version: 1.1.8
+  version: 1.1.9
 platform: ruby
 authors:
 - Adolfo Villafiorita
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-11-12 00:00:00.000000000 Z
+date: 2022-01-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: apache_log-parser
@@ -38,6 +38,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: ipaddr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: iso_country_codes
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -66,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Generate requests reports in HTML, OrgMode, and SQLite format from an
   Apache log file.
 email:
@@ -90,6 +132,7 @@ files:
 - lib/apache_log_report/apache_log_report.rb
 - lib/apache_log_report/data_cruncher.rb
 - lib/apache_log_report/emitter.rb
+- lib/apache_log_report/ip_locator.rb
 - lib/apache_log_report/log_parser.rb
 - lib/apache_log_report/options_parser.rb
 - lib/apache_log_report/templates/_output_table.html.erb
@@ -104,7 +147,10 @@ metadata:
   homepage_uri: https://www.ict4g.net/gitea/adolfo/apache_log_report
   source_code_uri: https://www.ict4g.net/gitea/adolfo/apache_log_report
   changelog_uri: https://www.ict4g.net/gitea/adolfo/apache_log_report/CHANGELOG.org
-post_install_message:
+post_install_message: |2
+    !    The 'apache_log_report' gem has been deprecated and has been replaced 'log_sense'.
+    !    See: https://rubygems.org/gems/log_sense
+    !    And: https://www.ict4g.net/gitea/adolfo/log_sense
 rdoc_options: []
 require_paths:
 - lib
@@ -119,7 +165,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.3.1
 signing_key:
 specification_version: 4
 summary: Generate analytics from an Apache log file.