log_sense 2.1.0 → 2.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e17413654dc552a716548d7a55e127c5f0703cf29643d97d500e0cf51b20eba3
-  data.tar.gz: 56765b54fdda4f64add3fbb97ea3b4073976d25f4cb67cc932a7d350d85793cb
+  metadata.gz: c2acfa576d18214e05df5520518000955d12957756674c26a5fd63ac6efc9969
+  data.tar.gz: 781b38f2a7118efe8f0c1ca0cba6d21bdf0a77bd3fe27a8c1dc27ca5cdc2b3e7
 SHA512:
-  metadata.gz: dc77b1c63c08a9a190aae0ab0ab44bdffe37e0dc2e999d3dc8a236703dc8e430fb32ff09d884d5d9e90ff7870b7bb212d0cc486e877982ca3c5328a3aeda5943
-  data.tar.gz: e73ad267ad54dc4e3c09481ec4f055e6d9e1e9346262b04975b754bd8822b19ce27298299ce6fb7cc3511e5248b6303392c984b48d68d40bd398bd14d7be9d80
+  metadata.gz: bc877d0198d6f170addfd9a0972ed7df57df2d5c00fdac850808ab1f8f86c3ed3d2aa37cc1058f1c6ca1da86bbdc6e6e09a44778edca3a48aaf825f4d5aefa26
+  data.tar.gz: ed99ed592a9a689fac9d319b1f346e705fcbbde021f45332d219097ed0df351d6b1d8a7bf2e0a72307b9932e2ab63ece7d0d8c2f2ef4fb9be45d14e2a5c6c863

data/CHANGELOG.org

@@ -2,6 +2,15 @@
 #+AUTHOR: Adolfo Villafiorita
 #+STARTUP: showall
 
+* 2.2.1
+
+- [Bug] Small fixes  
+
+* 2.2.0
+
+- [User] Better management of Delayed Job (show also completed)
+- [Refactoring] Rails rxegexps are now pre-compiled with /o
+  
 * 2.1.0
 
 - [User] Delayed Job Errors

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    log_sense (2.0.0)
+    log_sense (2.2.0)
       browser (~> 5.3.0)
       ipaddr (~> 1.2.0)
       iso_country_codes (~> 0.7.0)

data/exe/log_sense

@@ -168,4 +168,6 @@ else
 
   warn "Emitting..." if @options[:verbose]
   puts LogSense::Emitter.emit @reports, @data, @options
+
+  exit 0
 end

data/lib/log_sense/rails/log_parser.rb

@@ -165,6 +165,9 @@ module LogSense
         # hash
         pending = {}
 
+        # for delayed jobs
+        pending_jobs = {}
+
         # Fatal explanation messages span several lines (2, 4, ?)
         #
         # We keep a Hash with the FATAL explanation messages and we persist when
@@ -304,15 +307,79 @@ module LogSense
               next
             end
 
+            #
+            # Match enqueuing job
+            #
+            data = match_and_process_enqueuing_job line
+            if data
+              id = data[:job_id]
+              pending_jobs[id] = data.merge(pending_jobs[id] || {})
+              next
+            end
+
+            #
+            # Match running
+            # 
+            data = match_and_process_running_job line
+            if data
+              id = data[:job_id]
+              # change the key to pid
+              pid = data[:object_id]
+              pending_jobs[pid] = data.merge(pending_jobs[id] || {})
+
+              pending_jobs.delete(id)
+              next
+            end
+
+            #
+            # Match completed
+            # 
+            data = match_and_process_completed_job line
+            if data
+              id = data[:object_id]
+              # it has to be there!
+              if pending_jobs[id]
+                data = (pending_jobs[id] || {}).merge(data)
+              end
+              
+              ins_job.execute(
+                data[:started_at],
+                data[:ended_at],
+                data[:duration_total_ms],
+                data[:worker],
+                data[:host],
+                data[:pid],
+                data[:log_id],
+                data[:job_id], # no longer necessary
+                data[:object_id], # completed jobs are destroyed
+                data[:method],
+                data[:arguments],
+                data[:exit_status],
+                data[:attempt],
+                data[:error_msg],
+                filename,
+                line_number
+              )
+              pending_jobs.delete(id)
+              next
+            end
+
             #
             # Match job errors
             #
             data = match_and_process_job_error line
             if data
+              id = data[:object_id]
+              # TODO: no need for if (here and everywhere else)
+              # it has to be there!
+              if pending_jobs[id]
+                data = (pending_jobs[id] || {}).merge(data)
+              end
+              
               ins_job.execute(
-                data[:ended_at], # this is temporary (while we wait to parse BEGIN) + required by filter
+                data[:started_at],
                 data[:ended_at],
-                0,
+                nil,
                 data[:worker],
                 data[:host],
                 data[:pid],
@@ -327,6 +394,8 @@ module LogSense
                 filename,
                 line_number
               )
+
+              pending_jobs.delete(id)
             end
           end
         end
@@ -337,23 +406,37 @@ module LogSense
           ins_error.execute(value)
         end
 
+        # DO NOT persist the pending_jobs which have not yet completed (those
+        # still available at: pending_jobs).
+        #
+        # In fact various entries initiated with RUNNING end up with "performed"
+        # (rather than COMPLETED).  Notice that entries COMPLETED always have a
+        # peformed entry as well.
+        #
+        # Since we do not yet process "performed" entry log and in pending jobs
+        # we end up accumulating a bunch of entries which are marked as
+        # "performed"
+        #
+        # Performed entries are tricky since they use JOB_ID, rather than the
+        # object_id and probably requires to change how we enter pending_ids
+
         db
       end
 
       # could be private here, I guess we keep them public to make them simpler
       # to try from irb
       
-      TIMESTAMP = /(?<timestamp>[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+)/
-      LOG_ID = /(?<log_id>[a-z0-9-]+)/
-      VERB = /(?<verb>GET|POST|PATCH|PUT|DELETE)/
-      URL = /(?<url>[^"]+)/
-      IP = /(?<ip>[0-9.]+)/
-      STATUS = /(?<status>[0-9]+)/
-      STATUS_IN_WORDS = /(OK|Unauthorized|Found|Internal Server Error|Bad Request|Method Not Allowed|Request Timeout|Not Implemented|Bad Gateway|Service Unavailable)/
-      MSECS = /[0-9.]+/
+      TIMESTAMP = '(?<timestamp>[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\\.[0-9]+)'
+      LOG_ID = '(?<log_id>[a-z0-9-]+)'
+      VERB = '(?<verb>GET|POST|PATCH|PUT|DELETE)'
+      URL = '(?<url>[^"]+)'
+      IP = '(?<ip>[0-9.]+)'
+      STATUS = '(?<status>[0-9]+)'
+      STATUS_IN_WORDS = '(OK|Unauthorized|Found|Internal Server Error|Bad Request|Method Not Allowed|Request Timeout|Not Implemented|Bad Gateway|Service Unavailable)'
+      MSECS = '[0-9.]+'
 
       # I, [2021-10-19T08:16:34.343858 #10477]  INFO -- : [67103c0d-455d-4fe8-951e-87e97628cb66] Started GET "/grow/people/471" for 217.77.80.35 at 2021-10-19 08:16:34 +0000
-      STARTED_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{LOG_ID}\] Started #{VERB} "#{URL}" for #{IP} at/
+      STARTED_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{LOG_ID}\] Started #{VERB} "#{URL}" for #{IP} at/o
 
       def match_and_process_start(line)
         matchdata = STARTED_REGEXP.match line
@@ -372,14 +455,14 @@ module LogSense
       # I, [2021-10-19T08:16:34.712331 #10477]  INFO -- : [67103c0d-455d-4fe8-951e-87e97628cb66] Completed 200 OK in 367ms (Views: 216.7ms | ActiveRecord: 141.3ms | Allocations: 168792)
       # I, [2021-12-09T16:53:52.657727 #2735058]  INFO -- : [0064e403-9eb2-439d-8fe1-a334c86f5532] Completed 200 OK in 13ms (Views: 11.1ms | ActiveRecord: 1.2ms)
       # I, [2021-12-06T14:28:19.736545 #2804090]  INFO -- : [34091cb5-3e7b-4042-aaf8-6c6510d3f14c] Completed 500 Internal Server Error in 66ms (ActiveRecord: 8.0ms | Allocations: 24885)
-      COMPLETED_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{LOG_ID}\] Completed #{STATUS} #{STATUS_IN_WORDS} in (?<total>#{MSECS})ms \((Views: (?<views>#{MSECS})ms \| )?ActiveRecord: (?<arec>#{MSECS})ms( \| Allocations: (?<alloc>[0-9]+))?\)/
+      COMPLETED_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{LOG_ID}\] Completed #{STATUS} #{STATUS_IN_WORDS} in (?<total>#{MSECS})ms \((Views: (?<views>#{MSECS})ms \| )?ActiveRecord: (?<arec>#{MSECS})ms( \| Allocations: (?<alloc>[0-9]+))?\)/o
 
       def match_and_process_completed(line)
         matchdata = (COMPLETED_REGEXP.match line)
         # exit_status = matchdata[:status].to_i == 500 ? "E" : "I"
         if matchdata
           {
-            exit_status: "I",
+            exit_status: "S:COMPLETED",
             ended_at: matchdata[:timestamp],
             log_id: matchdata[:log_id],
             status: matchdata[:status],
@@ -393,7 +476,7 @@ module LogSense
       end
 
       # I, [2021-10-19T08:16:34.345162 #10477]  INFO -- : [67103c0d-455d-4fe8-951e-87e97628cb66] Processing by PeopleController#show as HTML
-      PROCESSING_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{LOG_ID}\] Processing by (?<controller>[^ ]+) as/
+      PROCESSING_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{LOG_ID}\] Processing by (?<controller>[^ ]+) as/o
 
       def match_and_process_processing_by line
         matchdata = PROCESSING_REGEXP.match line
@@ -410,13 +493,13 @@ module LogSense
       # F, [2021-12-04T00:34:05.839209 #2735058] FATAL -- : [3a16162e-a6a5-435e-a9d8-c4df5dc0f728]   
       # F, [2021-12-04T00:34:05.839269 #2735058] FATAL -- : [3a16162e-a6a5-435e-a9d8-c4df5dc0f728] actionpack (5.2.4.4) lib/action_dispatch/middleware/debug_exceptions.rb:65:in `call'
 
-      FATAL_REGEXP = /F, \[#{TIMESTAMP} #[0-9]+\] FATAL -- : \[#{LOG_ID}\]/
+      FATAL_REGEXP = /F, \[#{TIMESTAMP} #[0-9]+\] FATAL -- : \[#{LOG_ID}\]/o
 
       def match_and_process_fatal(line)
         matchdata = FATAL_REGEXP.match line
         if matchdata
           {
-            exit_status: "F",
+            exit_status: "S:FAILED",
             log_id: matchdata[:log_id],
           }
         end
@@ -440,8 +523,8 @@ module LogSense
       # [f57e3648-568a-48f9-ae3a-a522b1ff3298] app/models/donations/donation.rb:440:in `build_items_for_delivery'
       # [f57e3648-568a-48f9-ae3a-a522b1ff3298] app/controllers/donations_controller.rb:1395:in `create_delivery'
 
-      EXCEPTION = /[A-Za-z_0-9:]+(Error|NotFound|Invalid|Unknown|Missing|ENOSPC)/
-      FATAL_EXPLANATION_REGEXP = /^\[#{LOG_ID}\] (?<context>#{EXCEPTION})?(?<description>.*)/
+      EXCEPTION = "[A-Za-z_0-9:]+(Error|NotFound|Invalid|Unknown|Missing|ENOSPC)"
+      FATAL_EXPLANATION_REGEXP = /^\[#{LOG_ID}\] (?<context>#{EXCEPTION})?(?<description>.*)/o
       def match_and_process_fatal_explanation(line)
         matchdata = FATAL_EXPLANATION_REGEXP.match line
         if matchdata
@@ -454,7 +537,7 @@ module LogSense
       end
 
       # I, [2024-07-01T02:21:34.339058 #1392909]  INFO -- : [815b3e28-8d6e-4741-8605-87654a9ff58c] BrowserInfo: "Unknown Browser","unknown_platform","Unknown","Devise::SessionsController","new","html","4db749654a0fcacbf3868f87723926e7405262f8d596e8514f4997dc80a3cd7e","2024-07-01T02:21:34+02:00"
-      BROWSER_INFO_REGEXP = /BrowserInfo: "(?<browser>.+)","(?<platform>.+)","(?<device_name>.+)","(?<controller>.+)","(?<method>.+)","(?<request_format>.+)","(?<anon_ip>.+)","(?<timestamp>.+)"/
+      BROWSER_INFO_REGEXP = /BrowserInfo: "(?<browser>.+)","(?<platform>.+)","(?<device_name>.+)","(?<controller>.+)","(?<method>.+)","(?<request_format>.+)","(?<anon_ip>.+)","(?<timestamp>.+)"/o
 
       def match_and_process_browser_info(line)
         matchdata = BROWSER_INFO_REGEXP.match line
@@ -474,12 +557,14 @@ module LogSense
 
       # Sequence:
       #
-      # - enqueued
-      # - running
-      # - performing
+      # - enqueued (LOG_ID user event, JOB_ID assigned by system)
+      # - running  (JOB_ID links to enqueued; PID assigned by system; OBJECT_ID assigned by the system)
+      # - performing (OBJECT_ID links to running;  OBJECT_ID assigned by the system; JOB_ID is new)
       # - (rendering)
-      # - performed
-      # - completed
+      # - performed (OBJECT_ID  links to running; JOB_ID links to previous)
+      # - completed (OBJECT_ID  links to running; JOB_ID links to previous)
+      #
+      # SOMETIMES PERFORMED APPEARS WITH NO COMPLETED.
       #
       # I, [2024-08-01T06:21:16.302152 #3569287]  INFO -- : [96d14192-c7cc-48a9-9df7-3786de20b085] [ActiveJob] Enqueued ActionMailer::Parameterized::DeliveryJob (Job ID: 01e82c5c-fb42-4e5f-b0a7-6fa9512a9fb5) to DelayedJob(mailers) with arguments: "MessageMailer", "build_message", "deliver_now", {:project_id=>1, :email_to=>"activpentrutine@gmail.com", :hash=>{:event_name=>"download", :subject=>"Aviz BRAC-MEGA240176", :download=>#<GlobalID:0x00007f02d8e1ad98 @uri=#<URI::GID gid://btf3/Download/10652>>, :group=>#<GlobalID:0x00007f02d8e1a820 @uri=#<URI::GID gid://btf3/Organization/10061>>}, :locale=>:ro}
       #
@@ -489,41 +574,107 @@ module LogSense
       #
       # I, [2024-08-01T06:21:22.137863 #3563911]  INFO -- : [ActiveJob] [ActionMailer::Parameterized::DeliveryJob] [01e82c5c-fb42-4e5f-b0a7-6fa9512a9fb5] Performed ActionMailer::Parameterized::DeliveryJob (Job ID: 01e82c5c-fb42-4e5f-b0a7-6fa9512a9fb5) from DelayedJob(mailers) in 886.42ms
       #
+      #
+      # I, [2024-08-01T06:38:41.005687 #3563911]  INFO -- : 2024-08-01T06:38:41+0200: [Worker(delayed_job host:shair1 pid:3563911)] 1 jobs processed at 1.4476 j/s, 0 failed
+      #
+      #
       # I, [2024-08-01T06:21:22.141853 #3563911]  INFO -- : 2024-08-01T06:21:22+0200: [Worker(delayed_job host:shair1 pid:3563911)] Job ActionMailer::Parameterized::DeliveryJob [01e82c5c-fb42-4e5f-b0a7-6fa9512a9fb5] from DelayedJob(mailers) with arguments: ["MessageMailer", "build_message", "deliver_now", {"project_id"=>1, "email_to"=>"activpentrutine@gmail.com", "hash"=>{"event_name"=>"download", "subject"=>"Aviz BRAC-MEGA240176", "download"=>{"_aj_globalid"=>"gid://btf3/Download/10652"}, "group"=>{"_aj_globalid"=>"gid://btf3/Organization/10061"}, "_aj_symbol_keys"=>["event_name", "subject", "download", "group"]}, "locale"=>{"_aj_serialized"=>"ActiveJob::Serializers::SymbolSerializer", "value"=>"ro"}, "_aj_symbol_keys"=>["project_id", "email_to", "hash", "locale"]}] (id=212885) (queue=mailers) COMPLETED after 0.9067
 
       # Sequence with errors:
       # (two log entries per error)
       #
       # E, [2024-08-15T05:10:30.613623 #4150573] ERROR -- : [ActiveJob] [ActionMailer::Parameterized::DeliveryJob] [79ea42c0-d280-4cf9-b77e-65917d4bc9fc] Error performing ActionMailer::Parameterized::DeliveryJob (Job ID: 79ea42c0-d280-4cf9-b77e-65917d4bc9fc) from DelayedJob(mailers) in 462.62ms: Net::SMTPFatalError (553 Recipient domain not specified.
+
       # E, [2024-08-15T05:10:30.614189 #4150573] ERROR -- : 2024-08-15T05:10:30+0200: [Worker(delayed_job host:shair1 pid:4150573)] Job ActionMailer::Parameterized::DeliveryJob [79ea42c0-d280-4cf9-b77e-65917d4bc9fc] from DelayedJob(mailers) with arguments: ["MessageMailer", "build_message", "deliver_now", {"project_id"=>1, "email_to"=>"-", "hash"=>{"event_name"=>"download", "subject"=>"Aviz BvREWE240258.2", "download"=>{"_aj_globalid"=>"gid://btf3/Download/10877"}, "group"=>{"_aj_globalid"=>"gid://btf3/Organization/10060"}, "_aj_symbol_keys"=>["event_name", "subject", "download", "group"]}, "locale"=>{"_aj_serialized"=>"ActiveJob::Serializers::SymbolSerializer", "value"=>"ro"}, "_aj_symbol_keys"=>["project_id", "email_to", "hash", "locale"]}] (id=213242) (queue=mailers) FAILED (22 prior attempts) with Net::SMTPFatalError: 553 Recipient domain not specified.
       
-      TIMESTAMP_WITH_TZONE = /(?<timestamp_tzone>[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\+[0-9]+)/
-      ID = /(?<id>[0-9]+)/
-      WORKER = /Worker\((?<worker>.*) host:(?<host>.+) pid:(?<pid>[0-9]+)\)/
-      METHOD = /(?<method>[A-Za-z0-9:#]+)/
-      TIMES = /(?<attempt>[0-9]+)/
-      ERROR_MSG = /(?<error_msg>.+)/
-      ARGUMENTS = /(?<arguments>.+)/
+      TIMESTAMP_WITH_TZONE = '(?<timestamp_tzone>[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\\+[0-9]+)'
+      ID = '(?<id>[0-9]+)'
+      JOB_ID = '(?<job_id>[a-zA-Z0-9-]+)'
+      WORKER = 'Worker\\((?<worker>.+) host:(?<host>.+) pid:(?<pid>[0-9]+)\\)'
+      METHOD = '(?<method>[A-Za-z0-9:#_]+)'
+      TIMES = '(?<attempt>[0-9]+)'
+      ERROR_MSG = '(?<error_msg>.+)'
+      ARGUMENTS = '(?<arguments>.+)'
 
-      ERROR_MESSAGE = /E, \[#{TIMESTAMP} #[0-9]+\] ERROR -- : #{TIMESTAMP_WITH_TZONE}: \[#{WORKER}\] Job #{METHOD} \[#{LOG_ID}\] from .+ with arguments: \[#{ARGUMENTS}\] \(id=#{ID}\) \(queue=.*\) FAILED \(#{TIMES} prior attempts\) with #{ERROR_MSG}/
+      #
+      # these are together, since they return temporary data
+      #
+      ENQUEUEING = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : \[#{LOG_ID}\] \[ActiveJob\] Enqueued #{METHOD} \(Job ID: #{JOB_ID}\) to .* with arguments: #{ARGUMENTS}/o
 
-      ERROR_MESSAGE_SHORT = /E, \[#{TIMESTAMP} #[0-9]+\] ERROR -- : #{TIMESTAMP_WITH_TZONE}: \[#{WORKER}\] Job #{METHOD} \(id=#{ID}\) FAILED \(#{TIMES} prior attempts\) with #{ERROR_MSG}/
+      def match_and_process_enqueuing_job(line)
+        matchdata = ENQUEUEING.match line
+        if matchdata
+          {
+            log_id: matchdata[:log_id],
+            job_id: matchdata[:job_id]
+          }
+        end
+      end
+
+      RUNNING_MESSAGE = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : #{TIMESTAMP_WITH_TZONE}: \[#{WORKER}\] Job #{METHOD} \[#{JOB_ID}\] from .+ with arguments: \[#{ARGUMENTS}\] \(id=#{ID}\) \(queue=.*\) RUNNING/o
+
+      def match_and_process_running_job(line)
+        matchdata = RUNNING_MESSAGE.match line
+        if matchdata
+          {
+            started_at: matchdata[:timestamp],
+            job_id: matchdata[:job_id],
+            object_id: matchdata[:id],
+            pid: matchdata[:pid]
+          }
+        end
+      end
+
+      COMPLETED_MESSAGE = /I, \[#{TIMESTAMP} #[0-9]+\]  INFO -- : #{TIMESTAMP_WITH_TZONE}: \[#{WORKER}\] Job #{METHOD} \[#{JOB_ID}\] from .+ with arguments: \[#{ARGUMENTS}\] \(id=#{ID}\) \(queue=.*\) COMPLETED after (?<duration_total_ms>#{MSECS})/o
+
+      def match_and_process_completed_job(line)
+        matchdata = COMPLETED_MESSAGE.match line
+        if matchdata
+          {
+            ended_at: matchdata[:timestamp],
+            duration_total_ms: matchdata[:duration_total_ms],
+            id: matchdata[:id],
+            job_id: matchdata[:job_id],
+            worker: matchdata[:worker],
+            host: matchdata[:host],
+            pid: matchdata[:pid],
+            log_id: matchdata.named_captures["log_id"],
+            object_id: matchdata[:id],
+            method: matchdata[:method],
+            exit_status: 'S:COMPLETED',
+            arguments: matchdata[:arguments]
+          }
+        end
+      end      
+      
+      # similar to completed with I->E, INFO->ERROR, COMPLETED->FAILED and final message structure a bit different
+      ERROR_MESSAGE_PERMANENT = /E, \[#{TIMESTAMP} #[0-9]+\] ERROR -- : #{TIMESTAMP_WITH_TZONE}: \[#{WORKER}\] Job #{METHOD} \[#{JOB_ID}\] from .+ with arguments: \[#{ARGUMENTS}\] \(id=#{ID}\) \(queue=.*\) (?<error_msg>FAILED permanently because of #{TIMES} consecutive failures)/o
+
+      ERROR_MESSAGE = /E, \[#{TIMESTAMP} #[0-9]+\] ERROR -- : #{TIMESTAMP_WITH_TZONE}: \[#{WORKER}\] Job #{METHOD} \[#{JOB_ID}\] from .+ with arguments: \[#{ARGUMENTS}\] \(id=#{ID}\) \(queue=.*\) FAILED \(#{TIMES} prior attempts\) with #{ERROR_MSG}/o
+
+      ERROR_MESSAGE_SHORT = /E, \[#{TIMESTAMP} #[0-9]+\] ERROR -- : #{TIMESTAMP_WITH_TZONE}: \[#{WORKER}\] Job #{METHOD} \(id=#{ID}\) FAILED \(#{TIMES} prior attempts\) with #{ERROR_MSG}/o
 
       def match_and_process_job_error(line)
-        [ERROR_MESSAGE, ERROR_MESSAGE_SHORT].map do |regexp|
+        [ERROR_MESSAGE_PERMANENT, ERROR_MESSAGE, ERROR_MESSAGE_SHORT].map do |regexp|
           matchdata = regexp.match line
           if matchdata
+            exit_status = if matchdata[:error_msg].include?("permanently")
+                            "S:FAILED"
+                          else
+                            "S:ERROR"
+                          end
+
             return {
               ended_at: matchdata[:timestamp],
-              duration_total_ms: 0,
+              duration_total_ms: nil, # we could compute the time to failure
               worker: matchdata[:worker],
               host: matchdata[:host],
               pid: matchdata[:pid],
-              log_id: matchdata.named_captures["log_id"],
+              job_id: matchdata.named_captures["job_id"],
               object_id: matchdata[:id],
               method: matchdata[:method],
               arguments: matchdata.named_captures["arguments"],
-              exit_status: 'E',
+              exit_status:,
               attempt: matchdata[:attempt],
               error_msg: matchdata[:error_msg],
             }

data/lib/log_sense/rails_aggregator.rb

@@ -108,7 +108,7 @@ module LogSense
                  sum(iif(context NOT LIKE '%ActionController::RoutingError%', 1, 0)) as OtherErrors
                  FROM Event JOIN Error
                  ON event.log_id == error.log_id
-                 WHERE #{filter} and exit_status == 'F'
+                 WHERE #{filter} and exit_status == 'S:FAILED'
                  GROUP BY strftime("%Y-%m-%d", started_at)
       ).gsub("\n", "") || [[]]
 
@@ -121,7 +121,7 @@ module LogSense
                  event.log_id
                  FROM Event JOIN Error
                  ON event.log_id == error.log_id
-                 WHERE #{filter} and exit_status == 'F'
+                 WHERE #{filter} and exit_status == 'S:FAILED'
       ).gsub("\n", "") || [[]]
 
       @fatal_grouped = @db.execute %(
@@ -134,20 +134,24 @@ module LogSense
                 GROUP BY description
       ).gsub("\n", "") || [[]]
       
-      @job_error_plot = @db.execute %(
+      @job_plot = @db.execute %(
           SELECT strftime("%Y-%m-%d", ended_at) as Day,
-                 count(distinct(id)) as Errors
+                 sum(iif(exit_status == 'S:COMPLETED', 1, 0)) as Completed,
+                 sum(iif(exit_status == 'S:ERROR' or exit_status == 'S:FAILED', 1, 0)) as Errors
                  FROM Job
                  WHERE #{filter}
                  GROUP BY strftime("%Y-%m-%d", ended_at)
       ).gsub("\n", "") || [[]]
 
-      @job_error = @db.execute %(
-          SELECT strftime("%Y-%m-%d %H:%M", ended_at),
-                 worker,
-                 host,
+      # worker,
+      # host,
+      # pid,
+      @jobs = @db.execute %(
+          SELECT strftime("%Y-%m-%d %H:%M", started_at),
+                 duration_total_ms,
                  pid,
                  object_id,
+                 exit_status,
                  method,
                  arguments,
                  error_msg,
@@ -161,12 +165,13 @@ module LogSense
                 host,
                 pid,
                 object_id,
+                exit_status,
+                GROUP_CONCAT(DISTINCT(error_msg)),
                 method,
                 arguments,
-                error_msg,
                 max(attempt)
                 FROM Job
-                WHERE #{filter}
+                WHERE #{filter} and (exit_status == 'S:ERROR' or exit_status == 'S:FAILED')
                 GROUP BY object_id
       ).gsub("\n", "") || [[]]
 

data/lib/log_sense/rails_report_shaper.rb

@@ -182,7 +182,7 @@ module LogSense
                 name: 'Routing Errors',
                 data: fatal_plot.filter(row => row[0] != '').map(row => row[2]),
                 type: 'bar',
-                color: '#DEDEDE',
+                color: '#D0D0D0',
                 label: {
                   show: true,
                   position: 'top'
@@ -209,14 +209,27 @@ module LogSense
           rows: data[:fatal_grouped],
           col: "small-12 cell"
         },
+        browsers(data),
+        platforms(data),
+        ips(data),
+        countries(data),
+        ip_per_hour_report_spec(ips_per_hour(data[:ips_per_hour])),
+        session_report_spec(ips_detailed(data[:ips_per_day_detailed])),
         {
-          title: "Job Error",
-          header: %w[Date Worker Host PID ID Error Method Arguments Attempt],
-          column_alignment: %i[left left left left left left left left right],
-          column_width: ["10%", "5%", "5%", "5%", "5%", "20%", "20%", "20%", "10%"],
-          rows: data[:job_error],
+          title: "Jobs (Completed and Failed)",
+          explanation: %(
+            This report includes completed and failed jobs, parsing lines
+            marked as COMPLETED or ERROR/FAILED.
+
+            This excludes from the table entries marked as RUNNING and then
+            completed with "performed".
+          ),
+          header: %w[Date Duration PID ID Exit_Status Method Arguments Error_Msg Attempts],
+          column_alignment: %i[left right left left left left left left right],
+          column_width: ["10%", "5%", "5%", "5%", "5%", "15%", "25%", "25%", "5%"],
+          rows: data[:jobs],
           col: "small-12 cell",
-          echarts_extra: "var fatal_plot=#{data[:job_error_plot].to_json}",
+          echarts_extra: "var fatal_plot=#{data[:job_plot].to_json}",
           echarts_spec: "{
             toolbox: {
                feature: {
@@ -241,9 +254,19 @@ module LogSense
             },
             series: [
               {
-                name: 'Errors',
+                name: 'Completed',
                 data: fatal_plot.filter(row => row[0] != '').map(row => row[1]),
                 type: 'bar',
+                color: '#D0D0D0',
+                label: {
+                  show: true,
+                  position: 'top'
+                },
+              },
+              {
+                name: 'Errors',
+                data: fatal_plot.filter(row => row[0] != '').map(row => row[2]),
+                type: 'bar',
                 color: '#D30001',
                 label: {
                   show: true,
@@ -255,18 +278,12 @@ module LogSense
         },
         {
           title: "Job Errors (grouped)",
-          header: %w[Worker Host PID ID Error Method Arguments Attempt],
-          column_alignment: %i[left left left left left left left right],
-          column_width: ["5%", "5%", "5%", "5%", "20%", "30%", "20%", "10%"],
+          header: %w[Worker Host PID ID Exit_Status Error Method Arguments Attempts],
+          column_alignment: %i[left left left left left left left left right],
+          column_width: ["5%", "5%", "5%", "5%", "5%", "20%", "25%", "20%", "10%"],
           rows: data[:job_error_grouped],
           col: "small-12 cell"
-        },
-        browsers(data),
-        platforms(data),
-        ips(data),
-        countries(data),
-        ip_per_hour_report_spec(ips_per_hour(data[:ips_per_hour])),
-        session_report_spec(ips_detailed(data[:ips_per_day_detailed]))
+        }
       ]
     end
   end

data/lib/log_sense/report_shaper.rb

@@ -1,5 +1,6 @@
 module LogSense
   class ReportShaper
+    SESSION_URL_LIMIT = 300
     WORDS_SEPARATOR = ' · '
 
     # return { [ip,day] => { [ hits, list of urls ] } }
@@ -14,7 +15,11 @@ module LogSense
             date,
             hash[ip][date].size,
             hash[ip][date].uniq.size,
-            hash[ip][date].uniq.size < 100 ? hash[ip][date].uniq.join(WORDS_SEPARATOR) : "[too many]"
+            if hash[ip][date].uniq.size < SESSION_URL_LIMIT
+              hash[ip][date].uniq.join(WORDS_SEPARATOR)
+            else
+              "[too many]"
+            end
           ]
         end
       end

data/lib/log_sense/version.rb

@@ -1,3 +1,3 @@
 module LogSense
-  VERSION = "2.1.0"
+  VERSION = "2.2.1"
 end

data/todo.org

@@ -2,12 +2,12 @@
 
 * Todo (2024-08-01)
 
+** T Break rails log_parser in subclasses (one per log type)
+
 ** T Distinguish between exceptions, errors with no timestamps and errors with timestamps  :feature:
    - In rails reports some errors don't have a timestamp, whereas other have.
 
    1. Check whether we can always get an ID of an event
-   2. 
-
 
 ** T Move geolocation to aggregator, so that we can perform queries more efficiently  :refactoring:
 ** T Filter on dates                                                :feature:
@@ -18,9 +18,7 @@
 ** T error on data
    - The data reported by echarts on the number of hits of a controller
      differs from the data shown in the table?
-** T Sidebar foreground color in new apache report
-** T Dark style
-** T Remove dependency from Zurb Foundation (native css grid instead)
+** T Remove dependency from Zurb Foundation (native css grid instead) :refactoring:
 ** T refactor report specifications in their own class          :refactoring:
 ** T Add lines not parsed to the report                             :feature:
 ** T Using an empty log as input raises an error                    :feature:
@@ -314,3 +312,22 @@
    :ARCHIVE_CATEGORY: todo
    :ARCHIVE_TODO: REJECTED
    :END:
+
+** D Dark style
+   :PROPERTIES:
+   :ARCHIVE_TIME: 2024-08-23 Fri 16:25
+   :ARCHIVE_FILE: ~/Sources/ruby/log_sense/todo.org
+   :ARCHIVE_OLPATH: Todo (2024-08-01)
+   :ARCHIVE_CATEGORY: todo
+   :ARCHIVE_TODO: D
+   :END:
+
+** D Sidebar foreground color in new apache report
+   :PROPERTIES:
+   :ARCHIVE_TIME: 2024-08-23 Fri 16:25
+   :ARCHIVE_FILE: ~/Sources/ruby/log_sense/todo.org
+   :ARCHIVE_OLPATH: Todo (2024-08-01)
+   :ARCHIVE_CATEGORY: todo
+   :ARCHIVE_TODO: D
+   :END:
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: log_sense
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.2.1
 platform: ruby
 authors:
 - Adolfo Villafiorita
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-08-20 00:00:00.000000000 Z
+date: 2024-08-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: browser