apache_log_report 1.1.5 → 1.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c4cd01d24bc31eb2fe1107781a94578b43210fdc7d63dd65190643e637dc8da1
-  data.tar.gz: f1889cc37617847862274bd3d65fab82662d5c8e988129a227ed80b1fe4b7887
+  metadata.gz: ffde49bcae6f56e31b5ef7cccd88d5391127ee330264c188132f9ccbec94f421
+  data.tar.gz: 04c799c68ab676b176314b7ebe68cd3514faa48f853b9793e76d505d71342ae9
 SHA512:
-  metadata.gz: 520c9648ccd15f9a3bc0b1761989bb091bc81cf7f16eac6cf5bc5bb73c0aef3e47a03eafd85f468c08834dbf93163c4b750ef31d61b63d277f546afcd21082b2
-  data.tar.gz: a1833663a335da90513d1054117b9ca39b2f9802c021f7f75dda929312c00e32e7a3725901454c3a3f583839d0addbf511f08e421117f2317892fe1fe1f93628
+  metadata.gz: 1b5010ca3fa37d6dceb8ab48d0fc5f256713db19f63366602f7200219c9ec51656160e2e50901709179fac67633f5703eb137432c5cd289bc080f14dc675aebf
+  data.tar.gz: 377a4a516fe144f6167c83850c0037cb9d0c216090ad197b2ca22b94d6ab156d1c696ceb31ab5c0be6f06227e7ef62dfd8a23667b26a9919e56eff431c9c977c

data/Gemfile.lock

@@ -1,9 +1,10 @@
 PATH
   remote: .
   specs:
-    apache_log_report (1.1.2)
+    apache_log_report (1.1.8)
       apache_log-parser
       browser
+      ipaddr
       sqlite3
       terminal-table
 
@@ -12,6 +13,7 @@ GEM
   specs:
     apache_log-parser (3.1.2)
     browser (5.3.1)
+    ipaddr (1.2.3)
     rake (12.3.3)
     sqlite3 (1.4.2)
     terminal-table (3.0.2)

data/README.org

@@ -2,34 +2,6 @@
 #+AUTHOR: Adolfo Villafiorita
 #+STARTUP: showall
 
-* Introduction
+This Gem has been superseded by [[https://rubygems.org/gems/log_sense][Log Sense]], which produces better
+outputs and parses also rails logs.
 
-* Installation
-
-* Usage
-
-* Change Log
-
-See the [[file:CHANGELOG.org][CHANGELOG]] file.
-
-* Todo
-
-** TODO Graphs in HTML output
-** TODO Countries
-
-* Compatibility
-
-
-* Author and Contributors
-
-[[http://ict4g.net/adolfo][Adolfo Villafiorita]].
-
-* Known Bugs
-
-Some known bugs and an unknown number of unknown bugs.
-
-(See the open issues for the known bugs.)
-
-* License
-
-Distributed under the terms of the [[http://opensource.org/licenses/MIT][MIT License]].

data/Rakefile

@@ -1,2 +1,15 @@
 require "bundler/gem_tasks"
 task :default => :spec
+
+require 'rake/testtask'
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+end
+
+require_relative './lib/apache_log_report/ip_locator.rb'
+
+desc "Convert Geolocation DB to sqlite"
+task :dbip_to_sqlite3, [:filename] do |tasks, args|
+  filename = args[:filename]
+  ApacheLogReport::IpLocator::dbip_to_sqlite filename
+end

data/apache_log_report.gemspec

@@ -29,6 +29,18 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "apache_log-parser"
   spec.add_dependency "browser"
+  spec.add_dependency "ipaddr"
+  spec.add_dependency "iso_country_codes"
   spec.add_dependency "sqlite3"
   spec.add_dependency "terminal-table"
+
+  spec.add_development_dependency "minitest"
+
+  spec.post_install_message = <<-MESSAGE
+  !    The 'apache_log_report' gem has been deprecated and has been replaced 'log_sense'.
+  !    See: https://rubygems.org/gems/log_sense
+  !    And: https://www.ict4g.net/gitea/adolfo/log_sense
+  MESSAGE
+
+
 end

data/exe/apache_log_report

@@ -31,6 +31,7 @@ if @options[:output_format]  == "sqlite"
   b.finish
 else
   @data = ApacheLogReport::DataCruncher.crunch @db, @options
+  @data = ApacheLogReport::DataCruncher.geolocate @data
 
   @ended_at = Time.now
   @duration = @ended_at - @started_at

data/lib/apache_log_report/data_cruncher.rb

@@ -1,17 +1,39 @@
 module ApacheLogReport
   module DataCruncher
-
     #
     # take a sqlite3 database and analyze data
     #
+    # @ variables are automatically put in the returned data
+    #
 
     def self.crunch db, options = {}
-      @first_day      = db.execute "SELECT datetime from LogLine order by datetime limit 1"
-      @last_day       = db.execute "SELECT datetime from LogLine order by datetime desc limit 1"
+      first_day_s = db.execute "SELECT datetime from LogLine order by datetime limit 1"
+      last_day_s  = db.execute "SELECT datetime from LogLine order by datetime desc limit 1"
+
+      # make first and last day into dates or nil
+      @first_day = first_day_s.empty? ? nil : Date.parse(first_day_s[0][0])
+      @last_day = last_day_s.empty? ? nil : Date.parse(last_day_s[0][0])
+
+      @total_days = 0
+      if @first_day and @last_day
+        @total_days = (@last_day - @first_day).to_i
+      end
+
       @log_size       = db.execute "SELECT count(datetime) from LogLine"
       @crawlers_size  = db.execute "SELECT count(datetime) from LogLine where bot == 1"
       @selfpolls_size = db.execute "SELECT count(datetime) from LogLine where ip == '::1'"
 
+      @first_day_requested = options[:from_date]
+      @last_day_requested = options[:to_date]
+
+      @first_day_in_analysis = date_intersect options[:from_date], @first_day, :max
+      @last_day_in_analysis = date_intersect options[:to_date], @last_day, :min
+
+      @total_days_in_analysis = 0
+      if @first_day_in_analysis and @last_day_in_analysis
+        @total_days_in_analysis = (@last_day_in_analysis - @first_day_in_analysis).to_i
+      end
+
       #
       # generate the where clause corresponding to the command line options to filter data
       #
@@ -54,7 +76,6 @@ module ApacheLogReport
       @total_hits = db.execute "SELECT count(datetime) from LogLine where #{filter}"
       @total_unique_visitors = db.execute "SELECT count(distinct(unique_visitor)) from LogLine where #{filter}"
       @total_size = db.execute "SELECT #{human_readable_size} from LogLine where #{filter}"
-      @total_days = (Date.parse(@last_day[0][0]) - Date.parse(@first_day[0][0])).to_i
 
       @daily_distribution = db.execute "SELECT date(datetime), #{human_readable_day}, count(datetime), count(distinct(unique_visitor)), #{human_readable_size} from LogLine  where #{filter} group by date(datetime)"
       @time_distribution = db.execute "SELECT strftime('%H', datetime), count(datetime), count(distinct(unique_visitor)), #{human_readable_size} from LogLine  where #{filter} group by strftime('%H', datetime)"
@@ -80,19 +101,41 @@ module ApacheLogReport
 
       @browsers = db.execute "SELECT browser, count(browser), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by browser order by count(browser) desc"
       @platforms = db.execute "SELECT platform, count(platform), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by platform order by count(platform) desc"
-      @ips = db.execute "SELECT ip, count(ip), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by ip order by count(ip) desc limit #{options[:limit]}"
       @referers = db.execute "SELECT referer, count(referer), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by referer order by count(referer) desc limit #{options[:limit]}"
       
-      @streaks = db.execute "SELECT ip, substr(datetime, 1, 10), path from LogLine order by ip, datetime"
+      @ips = db.execute "SELECT ip, count(ip), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by ip order by count(ip) desc limit #{options[:limit]}"
 
+      @streaks = db.execute "SELECT ip, substr(datetime, 1, 10), path from LogLine order by ip, datetime"
       data = {}
+
       self.instance_variables.each do |variable|
         var_as_symbol = variable.to_s[1..-1].to_sym
         data[var_as_symbol] = eval(variable.to_s)
       end
       data
     end
-  end
 
+    # add country code to data[:ips]
+    def self.geolocate data
+      @location_db = IpLocator::load_db
+      data[:ips].each do |ip|
+        country_code = IpLocator::locate_ip ip[0], @location_db
+        ip << country_code
+      end
+      data
+    end
+
+    private
+
+    def self.date_intersect date1, date2, method
+      if date1 and date2
+        [date1, date2].send(method)
+      elsif date1
+        date1
+      else
+        date2
+      end
+    end
+  end
 end
 

data/lib/apache_log_report/ip_locator.rb

@@ -0,0 +1,42 @@
+require 'csv'
+require 'sqlite3'
+require 'ipaddr'
+require 'iso_country_codes'
+
+module ApacheLogReport
+  module IpLocator
+    DB_FILE = "ip_locations/dbip-country-lite.sqlite3"
+
+    def self.dbip_to_sqlite db_location
+      db = SQLite3::Database.new ":memory:"
+      db.execute "CREATE TABLE ip_location (
+        from_ip_n INTEGER,
+        from_ip TEXT,
+        to_ip TEXT,
+        country_code TEXT
+      )"
+
+      ins = db.prepare "INSERT INTO ip_location(from_ip_n, from_ip, to_ip, country_code) values (?, ?, ?, ?)"
+      CSV.foreach(db_location) do |row|
+        ip = IPAddr.new row[0]
+        ins.execute(ip.to_i, row[0], row[1], row[2])
+      end
+
+      # persist to file
+      ddb = SQLite3::Database.new(DB_FILE)
+      b = SQLite3::Backup.new(ddb, 'main', db, 'main')
+      b.step(-1) #=> DONE
+      b.finish
+    end
+
+    def self.load_db
+      SQLite3::Database.new DB_FILE
+    end
+
+    def self.locate_ip ip, db
+      ip_n = IPAddr.new(ip).to_i
+      res = db.execute "SELECT * FROM ip_location where from_ip_n <= #{ip_n} order by from_ip_n desc limit 1"
+      IsoCountryCodes.find(res[0][3]).name
+    end
+  end
+end

data/lib/apache_log_report/options_parser.rb

@@ -18,11 +18,11 @@ module ApacheLogReport
           args[:limit] = n
         end
 
-        opts.on("-bDATE", "--begin=DATE", DateTime, "Consider entries after or on DATE") do |n|
+        opts.on("-bDATE", "--begin=DATE", Date, "Consider entries after or on DATE") do |n|
           args[:from_date] = n
         end
 
-        opts.on("-eDATE", "--end=DATE", DateTime, "Consider entries before or on DATE") do |n|
+        opts.on("-eDATE", "--end=DATE", Date, "Consider entries before or on DATE") do |n|
           args[:to_date] = n
         end
 

data/lib/apache_log_report/templates/template.html.erb

@@ -31,7 +31,11 @@
                 "Browsers",
                 "Platforms",
                 "Referers",
-                "IPs"
+                "IPs",
+                "Geolocation",
+                "Streaks",
+                "Command Invocation",
+                "Performance"
               ].each do |item| %>
                 <li class="nav-item">
                   <a href="#<%= item.downcase.gsub(' ', '-') %>"><%= item %></a>
@@ -40,6 +44,14 @@
             </ul>
           </li>
         </ul>
+        <p>
+          Generated by<br />
+          <a href="https://www.ict4g.net/gitea/adolfo/apache_log_report">
+            Apache Log Report
+          </a> <br />
+          on <%= DateTime.now.strftime("%Y-%m-%d %H:%M") %>.<br />
+          The lean log analyzer.
+        </p>
       </nav>
 
       <section>
@@ -50,6 +62,22 @@
             <h2 id="summary">Summary</h2>
 
             <table class="table summary">
+              <tr>
+                <th>Input file</th>
+                <td><b><%= (data[:log_file] || "stdin") %></b></td>
+              </tr>
+              <tr>
+                <th class="period">Period Analyzed</th>
+                <td class="period">
+                  <%= data[:first_day_in_analysis] %>
+                  --
+                  <%= data[:last_day_in_analysis] %>
+                </td>
+              </tr>
+              <tr>
+                <th class="days">Days </th>
+                <td class="days"><%= data[:total_days_in_analysis] %></td>
+              </tr>
               <tr>
                 <th class="hits">Hits</th>
                 <td class="hits"><%= data[:total_hits][0][0] %></td>
@@ -62,18 +90,6 @@
                 <th class="tx">Tx</th>
                 <td class="tx"><%= data[:total_size][0][0] %></td>
               </tr>
-              <tr>
-                <th class="period">Period</th>
-                <td class="period">
-                  <%= data[:first_day][0][0] %>
-                  --
-                  <%= data[:last_day][0][0] %>
-                </td>
-              </tr>
-              <tr>
-                <th class="days">Days </th>
-                <td class="days"><%= data[:total_days] %></td>
-              </tr>
             </table>
           </article>
           <article class="column col-6">
@@ -85,6 +101,14 @@
                   <th>Input file</th>
                   <td><b><%= (data[:log_file] || "stdin") %></b></td>
                 </tr>
+                <tr>
+                  <th>Period in Log</th>
+                  <td><%= data[:first_day] %> -- <%= data[:last_day] %></td>
+                </tr>
+                <tr>
+                  <th>Total days</th>
+                  <td><%= data[:total_days] %></td>
+                </tr>
                 <tr>
                   <th>Log size</th>
                   <td><%= data[:log_size][0][0] %></td>
@@ -120,7 +144,7 @@
           { title: "Browsers", header: ["Browser", "Hits", "Visitors", "Size"], rows: data[:browsers] },
           { title: "Platforms", header: ["Platform", "Hits", "Visitors", "Size"], rows: data[:platforms] },
           { title: "Referers", header: ["Referers", "Hits", "Visitors", "Size"], rows: data[:referers], col: "col-12" },
-          { title: "IPs", header: ["IPs", "Hits", "Visitors", "Size"], rows: data[:ips] },
+          { title: "IPs", header: ["IPs", "Hits", "Visitors", "Size", "Country"], rows: data[:ips] },
           { },
         ]
         %>
@@ -139,6 +163,31 @@
           <% end %>
         </div>
 
+        <article>
+          <h2 id="geolocation">Geolocation</h2>
+          <table class="table">
+            <thead>
+              <tr>
+                <th>Country Code</th>
+                <th>Total Hits</th>
+                <th>Total Visitors</th>
+                <th>IPs</th>
+              </tr>              
+            </thead>
+            <tbody>
+              <%# IP, Hits, Visitors Size, Country%>
+              <% data[:ips].group_by { |x| x[4] }.each do |k, v| %>
+                <tr>
+                  <td><%= k %></td>
+                  <td><%= v.map { |x| x[1] }.inject(&:+) %></td>
+                  <td><%= v.map { |x| x[2] }.inject(&:+) %></td>
+                  <td><%= v.map { |x| x[0] }.join(", ") %></td>
+                </tr>
+              <% end %>
+            </tbody>
+          </table>
+        </article>
+
         <article>
           <h2 id="streaks">Streaks</h2>
 
@@ -146,7 +195,16 @@
             <thead>
               <tr>
                 <th>IP</th>
-                <th>Day and URL</th>
+                <th>
+                  <div class="columns">
+                    <div class="col-2 column">
+                      Day
+                    </div>
+                    <div class="col-10 column">
+                      Resources
+                    </div>
+                  </div>
+                </th>
               </tr>
             </thead>
             <tbody>
@@ -154,11 +212,28 @@
                 <tr>
                   <td class="ip"><%= ip %></td>
                   <td class="streaks">
-                    <% date_urls.group_by(&:first).each do |date, urls| %>
-                      <% urls.each do |url| %>
-                        <b><%= url[1] %>:</b> <%= url[2] %> <br />
-                      <% end %>
+                    <div class="columns">
+                    <% date_urls.group_by { |x| x[1] }.each do |date, urls| %>
+                      <div class="col-2 column">
+                        <%= date %>
+                      </div>
+                      <div class="col-10 column">
+                        <span class="res-title">HTML:</span>
+                        <ul>
+                          <% urls.map { |x| x[2] }.select { |x| x.match /.*\.html?/ }.each do |url| %>
+                            <li><%= url %></li>
+                          <% end %>
+                        </ul>
+                        
+                        <span class="res-title">Other Resources:</span>
+                        <ul>
+                          <% urls.map { |x| x[2] }.sort.select { |x| not x.match /.*\.html?/ }.each do |url| %>
+                            <li><%= url %></li>
+                          <% end %>
+                        </ul>
+                      </div>
                     <% end %>
+                    </div>
                   </td>
                 </tr>
               <% end %>
@@ -175,7 +250,7 @@
                 <tbody>
                   <tr>
                     <th>CLI Command</th>
-                    <td><pre><%= data[:command] %></pre></td>
+                    <td><code><%= data[:command] %></code></td>
                   </tr>
                   <tr>
                     <th>Input file</th>

data/lib/apache_log_report/templates/template.org.erb

@@ -10,7 +10,11 @@
 | Hits                | <%= "%10d" % data[:total_hits][0][0] %> |
 | Unique Visitors     | <%= "%10d" % data[:total_unique_visitors][0][0]  %> |
 | Tx                  | <%= "%10s" % data[:total_size][0][0] %> |
-| Days                | <%= "%10d" % data[:total_days][0][0]  %> |
+| Logged Period       | <%= data[:first_day]  %> -- <%= data[:last_day] %> |
+| Days                | <%= "%10d" % data[:total_days]  %> |
+| Period Requested    | <%= data[:first_day_requested] %> -- <%= data[:last_day_requested] %> |
+| Period Analyzed     | <%= data[:first_day_in_analysis] %> -- <%= data[:last_day_in_analysis] %> |
+| Days in Analysis    | <%= data[:total_days_in_analysis] %> |
 
 * Daily Distribution
 

data/lib/apache_log_report/version.rb

@@ -1,3 +1,3 @@
 module ApacheLogReport
-  VERSION = "1.1.5"
+  VERSION = "1.1.9"
 end

data/lib/apache_log_report.rb

@@ -2,4 +2,5 @@ require 'apache_log_report/version'
 require 'apache_log_report/options_parser'
 require 'apache_log_report/log_parser'
 require 'apache_log_report/data_cruncher'
+require 'apache_log_report/ip_locator'
 require 'apache_log_report/emitter'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: apache_log_report
 version: !ruby/object:Gem::Version
-  version: 1.1.5
+  version: 1.1.9
 platform: ruby
 authors:
 - Adolfo Villafiorita
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-11-01 00:00:00.000000000 Z
+date: 2022-01-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: apache_log-parser
@@ -38,6 +38,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: ipaddr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: iso_country_codes
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -66,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Generate requests reports in HTML, OrgMode, and SQLite format from an
   Apache log file.
 email:
@@ -90,6 +132,7 @@ files:
 - lib/apache_log_report/apache_log_report.rb
 - lib/apache_log_report/data_cruncher.rb
 - lib/apache_log_report/emitter.rb
+- lib/apache_log_report/ip_locator.rb
 - lib/apache_log_report/log_parser.rb
 - lib/apache_log_report/options_parser.rb
 - lib/apache_log_report/templates/_output_table.html.erb
@@ -104,7 +147,10 @@ metadata:
   homepage_uri: https://www.ict4g.net/gitea/adolfo/apache_log_report
   source_code_uri: https://www.ict4g.net/gitea/adolfo/apache_log_report
   changelog_uri: https://www.ict4g.net/gitea/adolfo/apache_log_report/CHANGELOG.org
-post_install_message:
+post_install_message: |2
+    !    The 'apache_log_report' gem has been deprecated and has been replaced 'log_sense'.
+    !    See: https://rubygems.org/gems/log_sense
+    !    And: https://www.ict4g.net/gitea/adolfo/log_sense
 rdoc_options: []
 require_paths:
 - lib
@@ -119,7 +165,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.22
+rubygems_version: 3.0.3.1
 signing_key:
 specification_version: 4
 summary: Generate analytics from an Apache log file.