log_sense 2.2.1 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c2acfa576d18214e05df5520518000955d12957756674c26a5fd63ac6efc9969
-  data.tar.gz: 781b38f2a7118efe8f0c1ca0cba6d21bdf0a77bd3fe27a8c1dc27ca5cdc2b3e7
+  metadata.gz: 8a9992db2fa42d1de3dac0bacbeed543a097e351c8bc9999c60febb3f67714a9
+  data.tar.gz: 06b9ed1329cbf3fdfdf889d154576dff85f16178e817b29fffede535eb1f12be
 SHA512:
-  metadata.gz: bc877d0198d6f170addfd9a0972ed7df57df2d5c00fdac850808ab1f8f86c3ed3d2aa37cc1058f1c6ca1da86bbdc6e6e09a44778edca3a48aaf825f4d5aefa26
-  data.tar.gz: ed99ed592a9a689fac9d319b1f346e705fcbbde021f45332d219097ed0df351d6b1d8a7bf2e0a72307b9932e2ab63ece7d0d8c2f2ef4fb9be45d14e2a5c6c863
+  metadata.gz: 0510a72e71e67e7bba948219f3ab35e7422c95a5a167815ccd30f589c172f19f7704e6e583364ac2493a7ff32c18df1316ef36d05f7e0346f582516b62d9f806
+  data.tar.gz: a374eea7ae662ac5be2458620baa4276bd7957871ccea67e82ba2987f941257ec2ffd1d1e7b62efd764093204976759470ccf7ca08a8152ecf1776d5eb023499

data/CHANGELOG.org

@@ -2,6 +2,10 @@
 #+AUTHOR: Adolfo Villafiorita
 #+STARTUP: showall
 
+* 2.3.0
+
+- Adds average response time by day
+
 * 2.2.1
 
 - [Bug] Small fixes  

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    log_sense (2.2.0)
+    log_sense (2.2.1)
       browser (~> 5.3.0)
       ipaddr (~> 1.2.0)
       iso_country_codes (~> 0.7.0)
@@ -11,7 +11,13 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
+    ast (2.4.2)
     browser (5.3.1)
+    bundle-audit (0.1.0)
+      bundler-audit
+    bundler-audit (0.9.2)
+      bundler (>= 1.2.0, < 3)
+      thor (~> 1.0)
     debug (1.9.2)
       irb (~> 1.10)
       reline (>= 0.3.8)
@@ -21,30 +27,59 @@ GEM
       rdoc (>= 4.0.0)
       reline (>= 0.4.2)
     iso_country_codes (0.7.8)
+    json (2.7.2)
+    language_server-protocol (3.17.0.3)
     mini_portile2 (2.8.7)
     minitest (5.24.1)
+    parallel (1.26.3)
+    parser (3.3.4.2)
+      ast (~> 2.4.1)
+      racc
     psych (5.1.2)
       stringio
+    racc (1.8.1)
+    rainbow (3.1.1)
     rake (13.2.1)
     rdoc (6.7.0)
       psych (>= 4.0.0)
+    regexp_parser (2.9.2)
     reline (0.5.9)
       io-console (~> 0.5)
+    rexml (3.3.6)
+      strscan
+    rubocop (1.65.1)
+      json (~> 2.3)
+      language_server-protocol (>= 3.17.0)
+      parallel (~> 1.10)
+      parser (>= 3.3.0.2)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 2.4, < 3.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.31.1, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 2.4.0, < 3.0)
+    rubocop-ast (1.32.1)
+      parser (>= 3.3.1.0)
+    ruby-progressbar (1.13.0)
     sqlite3 (2.0.3)
       mini_portile2 (~> 2.8.0)
     stringio (3.1.1)
+    strscan (3.1.0)
     terminal-table (3.0.2)
       unicode-display_width (>= 1.1.1, < 3)
+    thor (1.3.1)
     unicode-display_width (2.5.0)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
+  bundle-audit
   debug (~> 1.9.0)
   log_sense!
   minitest (~> 5.24.0)
   rake (~> 13.0)
+  rubocop
 
 BUNDLED WITH
    2.5.3

data/lib/log_sense/apache/log_line_parser.rb

@@ -21,33 +21,33 @@ module LogSense
     # 116.179.32.16 - - [19/Dec/2021:22:35:11 +0100] "GET / HTTP/1.1" 200 135 "-" "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)"
     #
     class LogLineParser
-      DAY = /[0-9]{2}/
-      MONTH = /[A-Za-z]{3}/
-      YEAR = /[0-9]{4}/
-      TIMEC = /[0-9]{2}/
-      TIMEZONE = /(\+|-)[0-9]{4}/
+      DAY = "[0-9]{2}"
+      MONTH = "[A-Za-z]{3}"
+      YEAR = "[0-9]{4}"
+      TIMEC = "[0-9]{2}"
+      TIMEZONE = "(\\+|-)[0-9]{4}"
 
-      IP = /(?<ip>[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}|::1|unknown)/
-      IDENT = /(?<ident>[^ ]+|-)/
-      USERID = /(?<userid>[^ ]+|-)/
+      IP = "(?<ip>[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}|::1|unknown)"
+      IDENT = "(?<ident>[^ ]+|-)"
+      USERID = "(?<userid>[^ ]+|-)"
 
-      TIMESTAMP = /(?<date>#{DAY}\/#{MONTH}\/#{YEAR}):(?<time>#{TIMEC}:#{TIMEC}:#{TIMEC} #{TIMEZONE})/
+      TIMESTAMP = "(?<date>#{DAY}\\/#{MONTH}\\/#{YEAR}):(?<time>#{TIMEC}:#{TIMEC}:#{TIMEC} #{TIMEZONE})"
 
-      HTTP_METHODS = /GET|HEAD|POST|PUT|DELETE|CONNECT|OPTIONS|TRACE|PATCH/
-      WEBDAV_METHODS = /COPY|LOCK|MKCOL|MOVE|PROPFIND|PROPPATCH|UNLOCK/
-      OTHER_METHODS = /SEARCH|REPORT|PRI|HEAD\/robots.txt/
-      METHOD = /(?<method>#{HTTP_METHODS}|#{WEBDAV_METHODS}|#{OTHER_METHODS})/
-      PROTOCOL = /(?<protocol>HTTP\/[0-9]\.[0-9]|-|.*)/
-      URL = /(?<url>[^ ]+)/
-      REFERER = /(?<referer>[^"]*)/
-      RETURN_CODE = /(?<status>[1-5][0-9][0-9])/
-      SIZE = /(?<size>[0-9]+|-)/
-      USER_AGENT = /(?<user_agent>[^"]*)/
+      HTTP_METHODS = "GET|HEAD|POST|PUT|DELETE|CONNECT|OPTIONS|TRACE|PATCH"
+      WEBDAV_METHODS = "COPY|LOCK|MKCOL|MOVE|PROPFIND|PROPPATCH|UNLOCK"
+      OTHER_METHODS = "SEARCH|REPORT|PRI|HEAD/robots.txt"
+      METHOD = "(?<method>#{HTTP_METHODS}|#{WEBDAV_METHODS}|#{OTHER_METHODS})"
+      PROTOCOL = "(?<protocol>HTTP\/[0-9]\.[0-9]|-|.*)"
+      URL = "(?<url>[^ ]+)"
+      REFERER = '(?<referer>[^"]*)'
+      RETURN_CODE = "(?<status>[1-5][0-9][0-9])"
+      SIZE = "(?<size>[0-9]+|-)"
+      USER_AGENT = '(?<user_agent>[^"]*)'
 
       attr_reader :format
 
       def initialize
-        @format = /#{IP} #{IDENT} #{USERID} \[#{TIMESTAMP}\] "(#{METHOD} #{URL} #{PROTOCOL}|-|.+)" #{RETURN_CODE} #{SIZE} "#{REFERER}" "#{USER_AGENT}"/
+        @format = /#{IP} #{IDENT} #{USERID} \[#{TIMESTAMP}\] "(#{METHOD} #{URL} #{PROTOCOL}|-|.+)" #{RETURN_CODE} #{SIZE} "#{REFERER}" "#{USER_AGENT}"/o
       end
 
       def parse(line)

data/lib/log_sense/apache_aggregator.rb

@@ -54,6 +54,7 @@ module LogSense
 
       @most_requested_pages = @db.execute resource_query(html, gs)
       @most_requested_resources = @db.execute resource_query(non_html, gs)
+
       @missed_pages = @db.execute resource_query(html, bs)
       @missed_resources = @db.execute resource_query(non_html, bs)
 

data/lib/log_sense/rails/log_parser.rb

@@ -94,7 +94,7 @@ module LogSense
           method TEXT,
           request_format TEXT,
           anon_ip TEXT,
-          timestamp TEXT
+          started_at TEXT
         )
         EOS
 
@@ -107,7 +107,7 @@ module LogSense
          method,
          request_format,
          anon_ip,
-         timestamp
+         started_at
         )
         values (?, ?, ?, ?, ?, ?, ?, ?)
         EOS
@@ -550,7 +550,7 @@ module LogSense
             method: matchdata[:method],
             request_format: matchdata[:request_format],
             anon_ip: matchdata[:anon_ip],
-            timestamp: matchdata[:timestamp],
+            started_at: matchdata[:timestamp],
           }
         end
       end

data/lib/log_sense/rails_aggregator.rb

@@ -1,4 +1,7 @@
+# frozen_string_literal: true
+
 module LogSense
+  # Aggregate data from Logs
   class RailsAggregator < Aggregator
     WORDS_SEPARATOR = ' · '
 
@@ -42,6 +45,18 @@ module LogSense
                  where #{filter}
                  group by controller order by controller).gsub("\n", "")
 
+      @performance_over_time = @db.execute %(
+          SELECT strftime("%Y-%m-%d", ended_at),
+                 count(controller),
+                 printf("%.2f", min(duration_total_ms)),
+                 printf("%.2f", avg(duration_total_ms)),
+                 printf("%.2f", max(duration_total_ms))
+                 from Event
+                 where #{filter}
+                 group by strftime("%Y-%m-%d", ended_at)
+                 order by strftime("%Y-%m-%d", ended_at)
+      ).gsub("\n", "")
+
       #
       # Use the performance information to build a tree map.
       # We then compute by device and show the treemap and the table by device

data/lib/log_sense/rails_report_shaper.rb

@@ -1,4 +1,7 @@
+# frozen_string_literal: true
+
 module LogSense
+  # Specification of the Rails Reports
   class RailsReportShaper < ReportShaper
     def shape(data)
       [
@@ -42,6 +45,7 @@ module LogSense
         time_distribution(data),
         total_statuses(data),
         daily_statuses(data),
+        performance_over_time(data, col: "small-12 cell"),
         {
           title: "Rails Performance",
           header: %w[Controller Hits Min Avg Max],

data/lib/log_sense/report_shaper.rb

@@ -86,6 +86,56 @@ module LogSense
       }
     end
 
+    def performance_over_time(data, colors: ["#d30001", "#888888"], col: "small-12 cell")
+      {
+        title: "Performance over Time",
+        header: %w[Date Count Min Avg Max],
+        column_alignment: %i[center right right right right],
+        rows: data[:performance_over_time],
+        col: col,
+        echarts_spec: "{
+            xAxis: {
+              type: 'category',
+              data: SERIES_DATA.map(row => row['Date'])
+            },
+            yAxis: [
+              {
+                type: 'value',
+                name: 'Average'
+              },
+              {
+                type: 'value',
+                name: 'Max'
+              }
+            ],
+            tooltip: {
+               trigger: 'axis'
+            },
+            series: [
+              {
+                data: SERIES_DATA.map(row => row['Avg']),
+                type: 'line',
+                color: '#{colors[0]}',
+                label: {
+                   show: true,
+                   position: 'top'
+                },
+              },
+              {
+                data: SERIES_DATA.map(row => row['Max']),
+                type: 'line',
+                color: '#{colors[1]}',
+                yAxisIndex: 1,
+                label: {
+                   show: true,
+                   position: 'top'
+                },
+              },
+            ]
+          }",
+      }
+    end
+
     #
     # Reports shared between rails and apache/nginx
     #

data/lib/log_sense/version.rb

@@ -1,3 +1,3 @@
 module LogSense
-  VERSION = "2.2.1"
+  VERSION = "2.3.0"
 end

data/log_sense.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |spec|
   spec.license       = "MIT"
   spec.required_ruby_version = Gem::Requirement.new(">= 2.6.9")
 
-  spec.metadata["allowed_push_host"] = "https://rubygems.org/"
+  # spec.metadata["allowed_push_host"] = "https://rubygems.org/"
 
   spec.metadata["homepage_uri"] = spec.homepage
   spec.metadata["source_code_uri"] = "https://github.com/shair-tech/log_sense/"
@@ -34,5 +34,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "terminal-table", "~> 3.0.0"
 
   spec.add_development_dependency "debug", "~> 1.9.0"
+  spec.add_development_dependency "rubocop"
+  spec.add_development_dependency "bundle-audit"
   spec.add_development_dependency "minitest", "~> 5.24.0"
 end

data/todo.org

@@ -1,7 +1,10 @@
 #+TITLE: Todo
 
 * Todo (2024-08-01)
-
+** T Show page with method
+   - we could generate a page with the result of the calls to a given method
+     (how many calls, distribution, ...). This would be the "show" of a rails app.
+     
 ** T Break rails log_parser in subclasses (one per log type)
 
 ** T Distinguish between exceptions, errors with no timestamps and errors with timestamps  :feature:

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: log_sense
 version: !ruby/object:Gem::Version
-  version: 2.2.1
+  version: 2.3.0
 platform: ruby
 authors:
 - Adolfo Villafiorita
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-08-23 00:00:00.000000000 Z
+date: 2025-02-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: browser
@@ -94,6 +94,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.9.0
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundle-audit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -172,7 +200,6 @@ homepage: https://github.com/shair-tech/log_sense/
 licenses:
 - MIT
 metadata:
-  allowed_push_host: https://rubygems.org/
   homepage_uri: https://github.com/shair-tech/log_sense/
   source_code_uri: https://github.com/shair-tech/log_sense/
   changelog_uri: https://github.com/shair-tech/log_sense/blob/main/CHANGELOG.org
@@ -191,7 +218,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.17
+rubygems_version: 3.5.22
 signing_key:
 specification_version: 4
 summary: Generate analytics for Rails and Apache/Nginx log file.