apache_log_report 1.1.7 → 1.1.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3ad003b9ae4768923ebb253dc937277372ff9adee264d77eb3db103153d8bcb5
-  data.tar.gz: f843e794726853611240ca3267cfed35666e3a04c030a78426bb524a8e40c632
+  metadata.gz: b9bf0afcc43a5985811da5287e1c44ebe86180f949b252a87b0e93fa37efb267
+  data.tar.gz: 252e6467f0d9b2acf912ea7def7c527abc735f9a5614f21cd1bd63431c43b8e8
 SHA512:
-  metadata.gz: 9a609336d73e83d15dd97939c1e1282ebe73b82d1b8e6d2cf07d2af769d4ad1e93803e825a38549c198d64f5f7e32637dcb6d3dea39eee14711e23b8adde35a0
-  data.tar.gz: 7fecc82af8cbb5debc54542f271855607479d1f142d1a725175afa51acbcc921f8955f4ba444594abc4e2cda4a6baf30421822c5d01e7d07bffcaf68eadbfe1e
+  metadata.gz: de105b5eef9ba1c492627725a87dbdb2b46a40b638547f1c128674cf6840459a2f853de33790279dfa8a7795fe8ed7da3e373b93000327cb96037df5af63c851
+  data.tar.gz: 55e7bf5c015ab07fe2df3f180b45532ff9feae1b08742f78b0138fc3065bbdd5a5ff6f88a0e38ded57fd98d71d13254c693fde1b0aff868ea6fd31cb819314ae

data/Gemfile.lock

@@ -1,9 +1,10 @@
 PATH
   remote: .
   specs:
-    apache_log_report (1.1.2)
+    apache_log_report (1.1.8)
       apache_log-parser
       browser
+      ipaddr
       sqlite3
       terminal-table
 
@@ -12,6 +13,7 @@ GEM
   specs:
     apache_log-parser (3.1.2)
     browser (5.3.1)
+    ipaddr (1.2.3)
     rake (12.3.3)
     sqlite3 (1.4.2)
     terminal-table (3.0.2)

data/README.org

@@ -2,115 +2,6 @@
 #+AUTHOR: Adolfo Villafiorita
 #+STARTUP: showall
 
-* Introduction
+This Gem has been superseded by [[https://rubygems.org/gems/log_sense][Log Sense]], which produces better
+outputs and parses also rails logs.
 
-ApacheLogReport generates reports and statistics from Apache web logs
-in the =combined= format.  Written in Ruby, it runs from the command
-line, it is fast, and it can be installed on any system which supports
-Ruby.
-
-ApacheLogReport moves along the lines of tools such as [[https://goaccess.io/][GoAccess]]
-and [[https://umami.is/][Umami]], focusing on privacy and data-ownership: the data
-generated by ApacheLogReport is stored on your computer and owned by
-you (like it should be).
-
-ApacheLogReport is also inspired by static websites generators:
-statistics are generated from the command line and accessed as static
-HTML files.  By generating static resources, ApacheLogReport
-significantly reduces the attack surface of your webserver and
-installation headaches.
-
-We have, for instance, a cron job running on our servers, generating
-statistics at night.  The generated files are then made available on a
-private area on the web.
-
-Statistics are generated from Apache log formats in the =combined=
-format.  Reports are tailored, but not limited, to web servers serving
-static websites.  No need to install Java Script code on your
-websites, no cookies installed, no user tracking.
-
-ApacheLogReport reports the following data:
-
-- Visitors, hits, unique visitors, bandwidth used
-- Most accessed HTML pages
-- Most accessed resources  
-- Response statuses
-- referers
-- OS, browsers, and devices
-- Streaks: resources accessed by a given IP over time
-- Potential attacks: access to resources which are not meant to be
-  served by a web server serving static websites
- 
-Filters from the command line allow to analyze specific periods and
-distinguish traffic generated by self polls and crawlers.
-
-ApacheLogReport generates HTML and SQLite outputs.  Moreover, it can
-also generate reports in Org Mode format, which can be then processed
-to various formats (including LaTeX and HTML).
-
-* Installation
-
-  #+begin_src bash
-  gem install apache_log_report
-  #+end_src
-
-* Usage
-
-  #+begin_src bash :results raw output :wrap example
-  apache_log_report --help
-  #+end_src
-
-  #+RESULTS:
-  #+begin_example
-  Usage: apache_log_report [options] [logfile]
-      -l, --limit=N                    Number of entries to show (defaults to 30)
-      -b, --begin=DATE                 Consider entries after or on DATE
-      -e, --end=DATE                   Consider entries before or on DATE
-      -i, --ignore-crawlers            Ignore crawlers
-      -p, --ignore-selfpoll            Ignore apaches self poll entries (from ::1)
-          --only-crawlers              Perform analysis on crawlers only
-      -u, --prefix=PREFIX              Prefix to add to all plots (used to run multiple analyses in the same dir)
-      -w, --suffix=SUFFIX              Suffix to add to all plots (used to run multiple analyses in the same dir)
-      -c, --code-export=WHAT           Control :export directive in Org Mode code blocks (code, results, *both*, none)
-      -f, --format=FORMAT              Output format: html, org, sqlite. Defaults to org mode
-      -v, --version                    Prints version information
-      -h, --help                       Prints this help
-  This is version 1.1.6
-  #+end_example
-
-* Change Log
-
-See the [[file:CHANGELOG.org][CHANGELOG]] file.
-
-* Todo
-
-** TODO Referers should only include the hostname?
-** TODO Graphs in HTML output
-** TODO Countries/IP Lookup
-** TODO Light and Dark Theme
-** TODO Embed CSS
-** TODO Declare datatypes in table outputs, so that we can format data
-
-* Compatibility
-
-ApacheLogReport should run on any system on which Ruby runs.
-
-Concerning the outputs:
-
-- The HTML report uses [[https://picturepan2.github.io/spectre/][Spectre.css]] and (will use) [[https://vega.github.io/vega-lite/][Vega Light]], which
-  are downloaded from a CDN
-- Org Mode plots data using [[http://gnuplot.info/][Gnuplot]]
-
-* Author and Contributors
-
-[[http://ict4g.net/adolfo][Adolfo Villafiorita]].
-
-* Known Bugs
-
-Some known bugs and an unknown number of unknown bugs.
-
-(See the open issues for the known bugs.)
-
-* License
-
-Distributed under the terms of the [[http://opensource.org/licenses/MIT][MIT License]].

data/Rakefile

@@ -1,2 +1,15 @@
 require "bundler/gem_tasks"
 task :default => :spec
+
+require 'rake/testtask'
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+end
+
+require_relative './lib/apache_log_report/ip_locator.rb'
+
+desc "Convert Geolocation DB to sqlite"
+task :dbip_to_sqlite3, [:filename] do |tasks, args|
+  filename = args[:filename]
+  ApacheLogReport::IpLocator::dbip_to_sqlite filename
+end

data/apache_log_report.gemspec

@@ -6,8 +6,8 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Adolfo Villafiorita"]
   spec.email         = ["adolfo.villafiorita@ict4g.net"]
 
-  spec.summary       = %q{Generate analytics from an Apache log file.}
-  spec.description   = %q{Generate requests reports in HTML, OrgMode, and SQLite format from an Apache log file.}
+  spec.summary       = %q{Superseded by Log Sense https://rubygems.org/gems/log_sense.}
+  spec.description   = %q{Generate requests reports in HTML, OrgMode, and SQLite format from an Apache log file. Superseded by Log Sense (https://rubygems.org/gems/log_sense).}
   spec.homepage      = "https://www.ict4g.net/gitea/adolfo/apache_log_report"
   spec.license       = "MIT"
   spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
@@ -29,6 +29,18 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "apache_log-parser"
   spec.add_dependency "browser"
+  spec.add_dependency "ipaddr"
+  spec.add_dependency "iso_country_codes"
   spec.add_dependency "sqlite3"
   spec.add_dependency "terminal-table"
+
+  spec.add_development_dependency "minitest"
+
+  spec.post_install_message = <<-MESSAGE
+  !    The 'apache_log_report' gem has been deprecated and has been replaced 'log_sense'.
+  !    See: https://rubygems.org/gems/log_sense
+  !    And: https://www.ict4g.net/gitea/adolfo/log_sense
+  MESSAGE
+
+
 end

data/exe/apache_log_report

@@ -31,6 +31,7 @@ if @options[:output_format]  == "sqlite"
   b.finish
 else
   @data = ApacheLogReport::DataCruncher.crunch @db, @options
+  @data = ApacheLogReport::DataCruncher.geolocate @data
 
   @ended_at = Time.now
   @duration = @ended_at - @started_at

data/lib/apache_log_report/data_cruncher.rb

@@ -1,7 +1,5 @@
-
 module ApacheLogReport
   module DataCruncher
-
     #
     # take a sqlite3 database and analyze data
     #
@@ -103,12 +101,13 @@ module ApacheLogReport
 
       @browsers = db.execute "SELECT browser, count(browser), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by browser order by count(browser) desc"
       @platforms = db.execute "SELECT platform, count(platform), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by platform order by count(platform) desc"
-      @ips = db.execute "SELECT ip, count(ip), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by ip order by count(ip) desc limit #{options[:limit]}"
       @referers = db.execute "SELECT referer, count(referer), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by referer order by count(referer) desc limit #{options[:limit]}"
       
-      @streaks = db.execute "SELECT ip, substr(datetime, 1, 10), path from LogLine order by ip, datetime"
+      @ips = db.execute "SELECT ip, count(ip), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by ip order by count(ip) desc limit #{options[:limit]}"
 
+      @streaks = db.execute "SELECT ip, substr(datetime, 1, 10), path from LogLine order by ip, datetime"
       data = {}
+
       self.instance_variables.each do |variable|
         var_as_symbol = variable.to_s[1..-1].to_sym
         data[var_as_symbol] = eval(variable.to_s)
@@ -116,6 +115,16 @@ module ApacheLogReport
       data
     end
 
+    # add country code to data[:ips]
+    def self.geolocate data
+      @location_db = IpLocator::load_db
+      data[:ips].each do |ip|
+        country_code = IpLocator::locate_ip ip[0], @location_db
+        ip << country_code
+      end
+      data
+    end
+
     private
 
     def self.date_intersect date1, date2, method
@@ -127,8 +136,6 @@ module ApacheLogReport
         date2
       end
     end
-
-
   end
 end
 

data/lib/apache_log_report/ip_locator.rb

@@ -0,0 +1,42 @@
+require 'csv'
+require 'sqlite3'
+require 'ipaddr'
+require 'iso_country_codes'
+
+module ApacheLogReport
+  module IpLocator
+    DB_FILE = "ip_locations/dbip-country-lite.sqlite3"
+
+    def self.dbip_to_sqlite db_location
+      db = SQLite3::Database.new ":memory:"
+      db.execute "CREATE TABLE ip_location (
+        from_ip_n INTEGER,
+        from_ip TEXT,
+        to_ip TEXT,
+        country_code TEXT
+      )"
+
+      ins = db.prepare "INSERT INTO ip_location(from_ip_n, from_ip, to_ip, country_code) values (?, ?, ?, ?)"
+      CSV.foreach(db_location) do |row|
+        ip = IPAddr.new row[0]
+        ins.execute(ip.to_i, row[0], row[1], row[2])
+      end
+
+      # persist to file
+      ddb = SQLite3::Database.new(DB_FILE)
+      b = SQLite3::Backup.new(ddb, 'main', db, 'main')
+      b.step(-1) #=> DONE
+      b.finish
+    end
+
+    def self.load_db
+      SQLite3::Database.new DB_FILE
+    end
+
+    def self.locate_ip ip, db
+      ip_n = IPAddr.new(ip).to_i
+      res = db.execute "SELECT * FROM ip_location where from_ip_n <= #{ip_n} order by from_ip_n desc limit 1"
+      IsoCountryCodes.find(res[0][3]).name
+    end
+  end
+end

data/lib/apache_log_report/templates/template.html.erb

@@ -32,6 +32,8 @@
                 "Platforms",
                 "Referers",
                 "IPs",
+                "Geolocation",
+                "Streaks",
                 "Command Invocation",
                 "Performance"
               ].each do |item| %>
@@ -142,7 +144,7 @@
           { title: "Browsers", header: ["Browser", "Hits", "Visitors", "Size"], rows: data[:browsers] },
           { title: "Platforms", header: ["Platform", "Hits", "Visitors", "Size"], rows: data[:platforms] },
           { title: "Referers", header: ["Referers", "Hits", "Visitors", "Size"], rows: data[:referers], col: "col-12" },
-          { title: "IPs", header: ["IPs", "Hits", "Visitors", "Size"], rows: data[:ips] },
+          { title: "IPs", header: ["IPs", "Hits", "Visitors", "Size", "Country"], rows: data[:ips] },
           { },
         ]
         %>
@@ -161,6 +163,31 @@
           <% end %>
         </div>
 
+        <article>
+          <h2 id="geolocation">Geolocation</h2>
+          <table class="table">
+            <thead>
+              <tr>
+                <th>Country Code</th>
+                <th>Total Hits</th>
+                <th>Total Visitors</th>
+                <th>IPs</th>
+              </tr>              
+            </thead>
+            <tbody>
+              <%# IP, Hits, Visitors Size, Country%>
+              <% data[:ips].group_by { |x| x[4] }.each do |k, v| %>
+                <tr>
+                  <td><%= k %></td>
+                  <td><%= v.map { |x| x[1] }.inject(&:+) %></td>
+                  <td><%= v.map { |x| x[2] }.inject(&:+) %></td>
+                  <td><%= v.map { |x| x[0] }.join(", ") %></td>
+                </tr>
+              <% end %>
+            </tbody>
+          </table>
+        </article>
+
         <article>
           <h2 id="streaks">Streaks</h2>
 
@@ -168,7 +195,16 @@
             <thead>
               <tr>
                 <th>IP</th>
-                <th>Day and URL</th>
+                <th>
+                  <div class="columns">
+                    <div class="col-2 column">
+                      Day
+                    </div>
+                    <div class="col-10 column">
+                      Resources
+                    </div>
+                  </div>
+                </th>
               </tr>
             </thead>
             <tbody>
@@ -176,11 +212,28 @@
                 <tr>
                   <td class="ip"><%= ip %></td>
                   <td class="streaks">
-                    <% date_urls.group_by(&:first).each do |date, urls| %>
-                      <% urls.each do |url| %>
-                        <b><%= url[1] %>:</b> <%= url[2] %> <br />
-                      <% end %>
+                    <div class="columns">
+                    <% date_urls.group_by { |x| x[1] }.each do |date, urls| %>
+                      <div class="col-2 column">
+                        <%= date %>
+                      </div>
+                      <div class="col-10 column">
+                        <span class="res-title">HTML:</span>
+                        <ul>
+                          <% urls.map { |x| x[2] }.select { |x| x.match /.*\.html?/ }.each do |url| %>
+                            <li><%= url %></li>
+                          <% end %>
+                        </ul>
+                        
+                        <span class="res-title">Other Resources:</span>
+                        <ul>
+                          <% urls.map { |x| x[2] }.sort.select { |x| not x.match /.*\.html?/ }.each do |url| %>
+                            <li><%= url %></li>
+                          <% end %>
+                        </ul>
+                      </div>
                     <% end %>
+                    </div>
                   </td>
                 </tr>
               <% end %>

data/lib/apache_log_report/version.rb

@@ -1,3 +1,3 @@
 module ApacheLogReport
-  VERSION = "1.1.7"
+  VERSION = "1.1.11"
 end

data/lib/apache_log_report.rb

@@ -2,4 +2,5 @@ require 'apache_log_report/version'
 require 'apache_log_report/options_parser'
 require 'apache_log_report/log_parser'
 require 'apache_log_report/data_cruncher'
+require 'apache_log_report/ip_locator'
 require 'apache_log_report/emitter'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: apache_log_report
 version: !ruby/object:Gem::Version
-  version: 1.1.7
+  version: 1.1.11
 platform: ruby
 authors:
 - Adolfo Villafiorita
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-11-01 00:00:00.000000000 Z
+date: 2022-01-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: apache_log-parser
@@ -38,6 +38,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: ipaddr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: iso_country_codes
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -66,8 +94,22 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Generate requests reports in HTML, OrgMode, and SQLite format from an
-  Apache log file.
+  Apache log file. Superseded by Log Sense (https://rubygems.org/gems/log_sense).
 email:
 - adolfo.villafiorita@ict4g.net
 executables:
@@ -90,6 +132,7 @@ files:
 - lib/apache_log_report/apache_log_report.rb
 - lib/apache_log_report/data_cruncher.rb
 - lib/apache_log_report/emitter.rb
+- lib/apache_log_report/ip_locator.rb
 - lib/apache_log_report/log_parser.rb
 - lib/apache_log_report/options_parser.rb
 - lib/apache_log_report/templates/_output_table.html.erb
@@ -104,7 +147,10 @@ metadata:
   homepage_uri: https://www.ict4g.net/gitea/adolfo/apache_log_report
   source_code_uri: https://www.ict4g.net/gitea/adolfo/apache_log_report
   changelog_uri: https://www.ict4g.net/gitea/adolfo/apache_log_report/CHANGELOG.org
-post_install_message:
+post_install_message: |2
+    !    The 'apache_log_report' gem has been deprecated and has been replaced 'log_sense'.
+    !    See: https://rubygems.org/gems/log_sense
+    !    And: https://www.ict4g.net/gitea/adolfo/log_sense
 rdoc_options: []
 require_paths:
 - lib
@@ -119,8 +165,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.3.1
 signing_key:
 specification_version: 4
-summary: Generate analytics from an Apache log file.
+summary: Superseded by Log Sense https://rubygems.org/gems/log_sense.
 test_files: []