log_sense 2.2.1 → 2.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c2acfa576d18214e05df5520518000955d12957756674c26a5fd63ac6efc9969
-  data.tar.gz: 781b38f2a7118efe8f0c1ca0cba6d21bdf0a77bd3fe27a8c1dc27ca5cdc2b3e7
+  metadata.gz: 43c38b704af22a839fedec504755be79abf4307c1183efe65ee7323984c4d6bc
+  data.tar.gz: ed1b73c4b9ccc5e6ffcae7682e67124aa82dba1d9fde9ab625764207893a11dd
 SHA512:
-  metadata.gz: bc877d0198d6f170addfd9a0972ed7df57df2d5c00fdac850808ab1f8f86c3ed3d2aa37cc1058f1c6ca1da86bbdc6e6e09a44778edca3a48aaf825f4d5aefa26
-  data.tar.gz: ed99ed592a9a689fac9d319b1f346e705fcbbde021f45332d219097ed0df351d6b1d8a7bf2e0a72307b9932e2ab63ece7d0d8c2f2ef4fb9be45d14e2a5c6c863
+  metadata.gz: 19857e10b0f5b72afdb491245101050a3348e267d6a9409416ab478e72b441ed8b559280f492f1c761753cbfd71abdafd5184f60095c53ca1bc0640a83c22f4a
+  data.tar.gz: 3f049b2b6756bfe9ef872021f6b34263736b12eb75ec2b06f086387550fbd87efe367ccc4348070cb492f5870b286c626fe6244ab19ea01f74684d301175b32b

data/CHANGELOG.org

@@ -2,6 +2,15 @@
 #+AUTHOR: Adolfo Villafiorita
 #+STARTUP: showall
 
+* 2.3.1
+
+- Updates geoip db
+- Updates dependencies    
+
+* 2.3.0
+
+- Adds average response time by day
+
 * 2.2.1
 
 - [Bug] Small fixes  

data/Gemfile.lock

@@ -1,50 +1,93 @@
 PATH
   remote: .
   specs:
-    log_sense (2.2.0)
-      browser (~> 5.3.0)
+    log_sense (2.3.1)
+      browser (~> 6.0.0)
       ipaddr (~> 1.2.0)
       iso_country_codes (~> 0.7.0)
-      sqlite3 (~> 2.0.0)
-      terminal-table (~> 3.0.0)
+      sqlite3 (~> 2.6.0)
+      terminal-table (~> 4.0.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    browser (5.3.1)
-    debug (1.9.2)
+    ast (2.4.3)
+    browser (6.0.0)
+    bundle-audit (0.1.0)
+      bundler-audit
+    bundler-audit (0.9.2)
+      bundler (>= 1.2.0, < 3)
+      thor (~> 1.0)
+    date (3.4.1)
+    debug (1.10.0)
       irb (~> 1.10)
       reline (>= 0.3.8)
-    io-console (0.7.2)
-    ipaddr (1.2.6)
-    irb (1.14.0)
+    io-console (0.8.0)
+    ipaddr (1.2.7)
+    irb (1.15.2)
+      pp (>= 0.6.0)
       rdoc (>= 4.0.0)
       reline (>= 0.4.2)
     iso_country_codes (0.7.8)
-    mini_portile2 (2.8.7)
-    minitest (5.24.1)
-    psych (5.1.2)
+    json (2.12.0)
+    language_server-protocol (3.17.0.5)
+    lint_roller (1.1.0)
+    mini_portile2 (2.8.9)
+    minitest (5.25.5)
+    parallel (1.27.0)
+    parser (3.3.8.0)
+      ast (~> 2.4.1)
+      racc
+    pp (0.6.2)
+      prettyprint
+    prettyprint (0.2.0)
+    prism (1.4.0)
+    psych (5.2.6)
+      date
       stringio
+    racc (1.8.1)
+    rainbow (3.1.1)
     rake (13.2.1)
-    rdoc (6.7.0)
+    rdoc (6.13.1)
       psych (>= 4.0.0)
-    reline (0.5.9)
+    regexp_parser (2.10.0)
+    reline (0.6.1)
       io-console (~> 0.5)
-    sqlite3 (2.0.3)
+    rubocop (1.75.5)
+      json (~> 2.3)
+      language_server-protocol (~> 3.17.0.2)
+      lint_roller (~> 1.1.0)
+      parallel (~> 1.10)
+      parser (>= 3.3.0.2)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 2.9.3, < 3.0)
+      rubocop-ast (>= 1.44.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 2.4.0, < 4.0)
+    rubocop-ast (1.44.1)
+      parser (>= 3.3.7.2)
+      prism (~> 1.4)
+    ruby-progressbar (1.13.0)
+    sqlite3 (2.6.0)
       mini_portile2 (~> 2.8.0)
-    stringio (3.1.1)
-    terminal-table (3.0.2)
-      unicode-display_width (>= 1.1.1, < 3)
-    unicode-display_width (2.5.0)
+    stringio (3.1.7)
+    terminal-table (4.0.0)
+      unicode-display_width (>= 1.1.1, < 4)
+    thor (1.3.2)
+    unicode-display_width (3.1.4)
+      unicode-emoji (~> 4.0, >= 4.0.4)
+    unicode-emoji (4.0.4)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  debug (~> 1.9.0)
+  bundle-audit
+  debug
   log_sense!
-  minitest (~> 5.24.0)
+  minitest
   rake (~> 13.0)
+  rubocop
 
 BUNDLED WITH
    2.5.3

data/lib/log_sense/apache/log_line_parser.rb

@@ -21,33 +21,33 @@ module LogSense
     # 116.179.32.16 - - [19/Dec/2021:22:35:11 +0100] "GET / HTTP/1.1" 200 135 "-" "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)"
     #
     class LogLineParser
-      DAY = /[0-9]{2}/
-      MONTH = /[A-Za-z]{3}/
-      YEAR = /[0-9]{4}/
-      TIMEC = /[0-9]{2}/
-      TIMEZONE = /(\+|-)[0-9]{4}/
+      DAY = "[0-9]{2}"
+      MONTH = "[A-Za-z]{3}"
+      YEAR = "[0-9]{4}"
+      TIMEC = "[0-9]{2}"
+      TIMEZONE = "(\\+|-)[0-9]{4}"
 
-      IP = /(?<ip>[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}|::1|unknown)/
-      IDENT = /(?<ident>[^ ]+|-)/
-      USERID = /(?<userid>[^ ]+|-)/
+      IP = "(?<ip>[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}|::1|unknown)"
+      IDENT = "(?<ident>[^ ]+|-)"
+      USERID = "(?<userid>[^ ]+|-)"
 
-      TIMESTAMP = /(?<date>#{DAY}\/#{MONTH}\/#{YEAR}):(?<time>#{TIMEC}:#{TIMEC}:#{TIMEC} #{TIMEZONE})/
+      TIMESTAMP = "(?<date>#{DAY}\\/#{MONTH}\\/#{YEAR}):(?<time>#{TIMEC}:#{TIMEC}:#{TIMEC} #{TIMEZONE})"
 
-      HTTP_METHODS = /GET|HEAD|POST|PUT|DELETE|CONNECT|OPTIONS|TRACE|PATCH/
-      WEBDAV_METHODS = /COPY|LOCK|MKCOL|MOVE|PROPFIND|PROPPATCH|UNLOCK/
-      OTHER_METHODS = /SEARCH|REPORT|PRI|HEAD\/robots.txt/
-      METHOD = /(?<method>#{HTTP_METHODS}|#{WEBDAV_METHODS}|#{OTHER_METHODS})/
-      PROTOCOL = /(?<protocol>HTTP\/[0-9]\.[0-9]|-|.*)/
-      URL = /(?<url>[^ ]+)/
-      REFERER = /(?<referer>[^"]*)/
-      RETURN_CODE = /(?<status>[1-5][0-9][0-9])/
-      SIZE = /(?<size>[0-9]+|-)/
-      USER_AGENT = /(?<user_agent>[^"]*)/
+      HTTP_METHODS = "GET|HEAD|POST|PUT|DELETE|CONNECT|OPTIONS|TRACE|PATCH"
+      WEBDAV_METHODS = "COPY|LOCK|MKCOL|MOVE|PROPFIND|PROPPATCH|UNLOCK"
+      OTHER_METHODS = "SEARCH|REPORT|PRI|HEAD/robots.txt"
+      METHOD = "(?<method>#{HTTP_METHODS}|#{WEBDAV_METHODS}|#{OTHER_METHODS})"
+      PROTOCOL = "(?<protocol>HTTP\/[0-9]\.[0-9]|-|.*)"
+      URL = "(?<url>[^ ]+)"
+      REFERER = '(?<referer>[^"]*)'
+      RETURN_CODE = "(?<status>[1-5][0-9][0-9])"
+      SIZE = "(?<size>[0-9]+|-)"
+      USER_AGENT = '(?<user_agent>[^"]*)'
 
       attr_reader :format
 
       def initialize
-        @format = /#{IP} #{IDENT} #{USERID} \[#{TIMESTAMP}\] "(#{METHOD} #{URL} #{PROTOCOL}|-|.+)" #{RETURN_CODE} #{SIZE} "#{REFERER}" "#{USER_AGENT}"/
+        @format = /#{IP} #{IDENT} #{USERID} \[#{TIMESTAMP}\] "(#{METHOD} #{URL} #{PROTOCOL}|-|.+)" #{RETURN_CODE} #{SIZE} "#{REFERER}" "#{USER_AGENT}"/o
       end
 
       def parse(line)

data/lib/log_sense/apache_aggregator.rb

@@ -54,6 +54,7 @@ module LogSense
 
       @most_requested_pages = @db.execute resource_query(html, gs)
       @most_requested_resources = @db.execute resource_query(non_html, gs)
+
       @missed_pages = @db.execute resource_query(html, bs)
       @missed_resources = @db.execute resource_query(non_html, bs)
 

data/lib/log_sense/options/parser.rb

@@ -139,7 +139,7 @@ module LogSense
 
           opts.on("-v", "--version", "Prints version information") do
             puts "log_sense version #{LogSense::VERSION}"
-            puts "Copyright (C) 2021-2024 Shair.Tech"
+            puts "Copyright (C) 2021-2025 Shair.Tech"
             puts "Distributed under the terms of the MIT license"
             exit
           end

data/lib/log_sense/rails/log_parser.rb

@@ -94,7 +94,7 @@ module LogSense
           method TEXT,
           request_format TEXT,
           anon_ip TEXT,
-          timestamp TEXT
+          started_at TEXT
         )
         EOS
 
@@ -107,7 +107,7 @@ module LogSense
          method,
          request_format,
          anon_ip,
-         timestamp
+         started_at
         )
         values (?, ?, ?, ?, ?, ?, ?, ?)
         EOS
@@ -550,7 +550,7 @@ module LogSense
             method: matchdata[:method],
             request_format: matchdata[:request_format],
             anon_ip: matchdata[:anon_ip],
-            timestamp: matchdata[:timestamp],
+            started_at: matchdata[:timestamp],
           }
         end
       end

data/lib/log_sense/rails_aggregator.rb

@@ -1,4 +1,7 @@
+# frozen_string_literal: true
+
 module LogSense
+  # Aggregate data from Logs
   class RailsAggregator < Aggregator
     WORDS_SEPARATOR = ' · '
 
@@ -42,6 +45,18 @@ module LogSense
                  where #{filter}
                  group by controller order by controller).gsub("\n", "")
 
+      @performance_over_time = @db.execute %(
+          SELECT strftime("%Y-%m-%d", ended_at),
+                 count(controller),
+                 printf("%.2f", min(duration_total_ms)),
+                 printf("%.2f", avg(duration_total_ms)),
+                 printf("%.2f", max(duration_total_ms))
+                 from Event
+                 where #{filter}
+                 group by strftime("%Y-%m-%d", ended_at)
+                 order by strftime("%Y-%m-%d", ended_at)
+      ).gsub("\n", "")
+
       #
       # Use the performance information to build a tree map.
       # We then compute by device and show the treemap and the table by device

data/lib/log_sense/rails_report_shaper.rb

@@ -1,4 +1,7 @@
+# frozen_string_literal: true
+
 module LogSense
+  # Specification of the Rails Reports
   class RailsReportShaper < ReportShaper
     def shape(data)
       [
@@ -42,6 +45,7 @@ module LogSense
         time_distribution(data),
         total_statuses(data),
         daily_statuses(data),
+        performance_over_time(data, col: "small-12 cell"),
         {
           title: "Rails Performance",
           header: %w[Controller Hits Min Avg Max],

data/lib/log_sense/report_shaper.rb

@@ -86,6 +86,56 @@ module LogSense
       }
     end
 
+    def performance_over_time(data, colors: ["#d30001", "#888888"], col: "small-12 cell")
+      {
+        title: "Performance over Time",
+        header: %w[Date Count Min Avg Max],
+        column_alignment: %i[center right right right right],
+        rows: data[:performance_over_time],
+        col: col,
+        echarts_spec: "{
+            xAxis: {
+              type: 'category',
+              data: SERIES_DATA.map(row => row['Date'])
+            },
+            yAxis: [
+              {
+                type: 'value',
+                name: 'Average'
+              },
+              {
+                type: 'value',
+                name: 'Max'
+              }
+            ],
+            tooltip: {
+               trigger: 'axis'
+            },
+            series: [
+              {
+                data: SERIES_DATA.map(row => row['Avg']),
+                type: 'line',
+                color: '#{colors[0]}',
+                label: {
+                   show: true,
+                   position: 'top'
+                },
+              },
+              {
+                data: SERIES_DATA.map(row => row['Max']),
+                type: 'line',
+                color: '#{colors[1]}',
+                yAxisIndex: 1,
+                label: {
+                   show: true,
+                   position: 'top'
+                },
+              },
+            ]
+          }",
+      }
+    end
+
     #
     # Reports shared between rails and apache/nginx
     #

data/lib/log_sense/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module LogSense
-  VERSION = "2.2.1"
+  VERSION = "2.3.1"
 end

data/log_sense.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |spec|
   spec.license       = "MIT"
   spec.required_ruby_version = Gem::Requirement.new(">= 2.6.9")
 
-  spec.metadata["allowed_push_host"] = "https://rubygems.org/"
+  # spec.metadata["allowed_push_host"] = "https://rubygems.org/"
 
   spec.metadata["homepage_uri"] = spec.homepage
   spec.metadata["source_code_uri"] = "https://github.com/shair-tech/log_sense/"
@@ -27,12 +27,14 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "browser", "~> 5.3.0"
+  spec.add_dependency "browser", "~> 6.0.0"
   spec.add_dependency "ipaddr", "~> 1.2.0"
   spec.add_dependency "iso_country_codes", "~> 0.7.0" 
-  spec.add_dependency "sqlite3", "~> 2.0.0"
-  spec.add_dependency "terminal-table", "~> 3.0.0"
+  spec.add_dependency "sqlite3", "~> 2.6.0"
+  spec.add_dependency "terminal-table", "~> 4.0.0"
 
-  spec.add_development_dependency "debug", "~> 1.9.0"
-  spec.add_development_dependency "minitest", "~> 5.24.0"
+  spec.add_development_dependency "debug"
+  spec.add_development_dependency "rubocop"
+  spec.add_development_dependency "bundle-audit"
+  spec.add_development_dependency "minitest"
 end

data/todo.org

@@ -1,7 +1,10 @@
 #+TITLE: Todo
 
 * Todo (2024-08-01)
-
+** T Show page with method
+   - we could generate a page with the result of the calls to a given method
+     (how many calls, distribution, ...). This would be the "show" of a rails app.
+     
 ** T Break rails log_parser in subclasses (one per log type)
 
 ** T Distinguish between exceptions, errors with no timestamps and errors with timestamps  :feature:

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: log_sense
 version: !ruby/object:Gem::Version
-  version: 2.2.1
+  version: 2.3.1
 platform: ruby
 authors:
 - Adolfo Villafiorita
-autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-08-23 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: browser
@@ -16,14 +15,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.3.0
+        version: 6.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.3.0
+        version: 6.0.0
 - !ruby/object:Gem::Dependency
   name: ipaddr
   requirement: !ruby/object:Gem::Requirement
@@ -58,56 +57,84 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 2.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 2.6.0
 - !ruby/object:Gem::Dependency
   name: terminal-table
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: 4.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: 4.0.0
 - !ruby/object:Gem::Dependency
   name: debug
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.9.0
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.9.0
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundle-audit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 5.24.0
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 5.24.0
+        version: '0'
 description: Generate analytics in HTML, txt, and SQLite format for Rails and Apache/Nginx
   log files.
 email:
@@ -172,11 +199,9 @@ homepage: https://github.com/shair-tech/log_sense/
 licenses:
 - MIT
 metadata:
-  allowed_push_host: https://rubygems.org/
   homepage_uri: https://github.com/shair-tech/log_sense/
   source_code_uri: https://github.com/shair-tech/log_sense/
   changelog_uri: https://github.com/shair-tech/log_sense/blob/main/CHANGELOG.org
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -191,8 +216,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.17
-signing_key:
+rubygems_version: 3.6.7
 specification_version: 4
 summary: Generate analytics for Rails and Apache/Nginx log file.
 test_files: []