karafka-web 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 741082cd49e03399e910d2119b2bbd5bffd397c0b548005db6dedc3d6e283557
-  data.tar.gz: b5b24528a2e489fa509c70deea9e79011d0945e6a6f1b8e44686acc0ec2ab96b
+  metadata.gz: 9edb576f473624295de9b0f1d62cbed131c9be1b480958a63c33f891fbf7cb19
+  data.tar.gz: 4f71e3a6ce1ca6eb423550824c5381e3b2b237c1b2af88c9108359bd225a7444
 SHA512:
-  metadata.gz: aba7fd6a0b9e3729219bd477319b6e06b0662e41f7734e0e43940df940f20451a63a7c4ab734475b29a1c615109329f04a4f6e9767c3cf36005b365ca2174b8f
-  data.tar.gz: '02160184e59088650bb6a00c831ef58a880b02a70dc2b2a4ccfceb4e140a0ee4e0b30e27a5e6df1b35e9516db47823464e6b125ed34ee29a742db4506460983c'
+  metadata.gz: 7e0f77ed2a8e6c902596d2fad16938c3117a49c3faa7d9c60e8018c24135b9c3f38e45ffe5a0aa6fffdbf6f514956d4122afbb90a021f0c6f6274e0cb5212e75
+  data.tar.gz: 6fc0272d850dcdd99c277dabb38c5a0761646709f679b794f5c4b8161eb9217439f087b7d3eba3763bec60902874413506c6492dbb9ee63a992a0b6674124689

data/CHANGELOG.md

@@ -1,5 +1,25 @@
 # Karafka Web changelog
 
+## 0.5.0 (2022-04-13)
+- [Improvement] Report job `-1001` offsets as `N/A` as in all the other places.
+- [Fix] Fix misspelling of word `committed`.
+- [Fix] Shutdown and revocation jobs statistics extraction crashes when idle initialized without messages (#53)
+
+### Upgrade notes
+
+Because of the reporting schema change, it is recommended to:
+
+- First, deploy **all** the Karafka consumer processes (`karafka server`)
+- Deploy the Web update to your web server.
+
+Please note that if you decide to use the updated Web UI with not updated consumers, you may hit a 500 error or offset related data may not be displayed correctly.
+
+## 0.4.1 (2023-04-12)
+- [Improvement] Replace the "x time ago" in the code explorer with an exact date (`2023-04-12 10:16:48.596 +0200 `).
+- [Improvement] When hovering over a message timestamp, a label with raw numeric timestamp will be presented.
+- [Improvement] Do not skip reporting on partitions subscribed that never received any messages.
+- [Fix] Skip reporting data on subscriptions that were revoked and not only stopped by us.
+
 ## 0.4.0 (2023-04-07)
 - [Improvement] Include active jobs and active partitions subscriptions count in the per-process tab navigation.
 - [Improvement] Include subscription groups names in the per-process subscriptions view.

data/Gemfile.lock

@@ -1,9 +1,9 @@
 PATH
   remote: .
   specs:
-    karafka-web (0.4.0)
+    karafka-web (0.5.0)
       erubi (~> 1.4)
-      karafka (>= 2.0.38, < 3.0.0)
+      karafka (>= 2.0.40, < 3.0.0)
       karafka-core (>= 2.0.12, < 3.0.0)
       roda (~> 3.63)
       tilt (~> 2.0)
@@ -26,7 +26,7 @@ GEM
     ffi (1.15.5)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
-    karafka (2.0.38)
+    karafka (2.0.40)
       karafka-core (>= 2.0.12, < 3.0.0)
       thor (>= 0.20)
       waterdrop (>= 2.4.10, < 3.0.0)

data/karafka-web.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |spec|
   spec.licenses    = %w[LGPL-3.0 Commercial]
 
   spec.add_dependency 'erubi', '~> 1.4'
-  spec.add_dependency 'karafka', '>= 2.0.38', '< 3.0.0'
+  spec.add_dependency 'karafka', '>= 2.0.40', '< 3.0.0'
   spec.add_dependency 'karafka-core', '>= 2.0.12', '< 3.0.0'
   spec.add_dependency 'roda', '~> 3.63'
   spec.add_dependency 'tilt', '~> 2.0'

data/lib/karafka/web/tracking/consumers/contracts/job.rb

@@ -14,9 +14,9 @@ module Karafka
             required(:started_at) { |val| val.is_a?(Float) && val >= 0 }
             required(:topic) { |val| val.is_a?(String) }
             required(:partition) { |val| val.is_a?(Integer) && val >= 0 }
-            required(:first_offset) { |val| val.is_a?(Integer) && val >= 0 }
-            required(:last_offset) { |val| val.is_a?(Integer) && val >= 0 }
-            required(:comitted_offset) { |val| val.is_a?(Integer) }
+            required(:first_offset) { |val| val.is_a?(Integer) && (val >= 0 || val == -1001) }
+            required(:last_offset) { |val| val.is_a?(Integer) && (val >= 0 || val == -1001) }
+            required(:committed_offset) { |val| val.is_a?(Integer) }
             required(:type) { |val| %w[consume revoked shutdown].include?(val) }
             required(:tags) { |val| val.is_a?(Karafka::Core::Taggable::Tags) }
           end

data/lib/karafka/web/tracking/consumers/listeners/errors.rb

@@ -69,10 +69,10 @@ module Karafka
               {
                 topic: consumer.topic.name,
                 consumer_group: consumer.topic.consumer_group.id,
-                partition: consumer.messages.metadata.partition,
-                first_offset: consumer.messages.first.offset,
-                last_offset: consumer.messages.last.offset,
-                comitted_offset: consumer.coordinator.seek_offset - 1,
+                partition: consumer.partition,
+                first_offset: consumer.messages.metadata.first_offset,
+                last_offset: consumer.messages.metadata.last_offset,
+                committed_offset: consumer.coordinator.seek_offset - 1,
                 consumer: consumer.class.to_s,
                 tags: consumer.tags
               }

data/lib/karafka/web/tracking/consumers/listeners/processing.rb

@@ -46,10 +46,17 @@ module Karafka
                          'revoked'
                        when 'consumer.shutdown.error'
                          'shutdown'
+                       # This is not a user facing execution flow, but internal system one
+                       # that is why it will not be reported as a separate job for the UI
+                       when 'consumer.idle.error'
+                         false
+                       else
+                         false
                        end
 
                 # job reference only exists for consumer work related operations.
                 # Only for them we need to deregister the job reference.
+                # This also refers only to consumer work that runs user operations.
                 return unless type
 
                 sampler.jobs.delete(
@@ -144,10 +151,10 @@ module Karafka
               {
                 started_at: float_now,
                 topic: consumer.topic.name,
-                partition: consumer.messages.metadata.partition,
-                first_offset: consumer.messages.first.offset,
-                last_offset: consumer.messages.last.offset,
-                comitted_offset: consumer.coordinator.seek_offset - 1,
+                partition: consumer.partition,
+                first_offset: consumer.messages.metadata.first_offset,
+                last_offset: consumer.messages.metadata.last_offset,
+                committed_offset: consumer.coordinator.seek_offset - 1,
                 consumer: consumer.class.to_s,
                 consumer_group: consumer.topic.consumer_group.id,
                 type: type,

data/lib/karafka/web/tracking/consumers/listeners/statistics.rb

@@ -87,13 +87,14 @@ module Karafka
             def partition_reportable?(pt_id, pt_stats)
               return false if pt_id == -1
 
-              # Skip until lag info is available
-              return false if pt_stats['consumer_lag'] == -1
-
               # Collect information only about what we are subscribed to and what we fetch or
-              # work in any way. Stopped means, we no longer work with it
+              # work in any way. Stopped means, we stopped working with it
               return false if pt_stats['fetch_state'] == 'stopped'
 
+              # Return if we no longer fetch this partition in a particular process. None means
+              # that we no longer have this subscription assigned and we do not fetch
+              return false if pt_stats['fetch_state'] == 'none'
+
               true
             end
 

data/lib/karafka/web/tracking/consumers/sampler.rb

@@ -14,7 +14,7 @@ module Karafka
           # Current schema version
           # This can be used in the future for detecting incompatible changes and writing
           # migrations
-          SCHEMA_VERSION = '1.1.0'
+          SCHEMA_VERSION = '1.2.0'
 
           # 60 seconds window for time tracked window-based metrics
           TIMES_TTL = 60

data/lib/karafka/web/ui/helpers/application_helper.rb

@@ -125,10 +125,31 @@ module Karafka
           # @param time [Float] UTC time float
           # @return [String] relative time tag for timeago.js
           def relative_time(time)
-            stamp = Time.at(time).getutc.iso8601
+            stamp = Time.at(time).getutc.iso8601(3)
             %(<time class="ltr" dir="ltr" title="#{stamp}" datetime="#{stamp}">#{time}</time>)
           end
 
+          # @param time [Time] time object we want to present with detailed ms label
+          # @return [String] span tag with raw timestamp as a title and time as a value
+          def time_with_label(time)
+            stamp = (time.to_f * 1000).to_i
+
+            %(<span title="#{stamp}">#{time}</span>)
+          end
+
+          # @param offset [Integer] offset
+          # @return [String] offset if correct or `N/A` with labeled explanation for offsets
+          #   that are less than 0. Offset with less than 0 indicates, that the offset was not
+          #   yet committed and there is no value we know of
+          def offset_with_label(offset)
+            if offset.negative?
+              title = 'Not available until first offset commit'
+              %(<span class="badge bg-secondary" title="#{title}">N/A</span>)
+            else
+              offset.to_s
+            end
+          end
+
           # Returns the view title html code
           #
           # @param title [String] page title

data/lib/karafka/web/ui/pro/views/consumers/consumer/_job.erb

@@ -16,13 +16,13 @@
     <code>#<%= job.type %></code>
   </td>
   <td>
-    <%= job.first_offset %>
+    <%== offset_with_label job.first_offset %>
   </td>
   <td>
-    <%= job.last_offset %>
+    <%== offset_with_label job.last_offset %>
   </td>
   <td>
-    <%= job.comitted_offset %>
+    <%== offset_with_label job.committed_offset %>
   </td>
   <td>
     <%== relative_time job.started_at %>

data/lib/karafka/web/ui/pro/views/consumers/consumer/_partition.erb

@@ -6,13 +6,7 @@
     <%= partition.id %>
   </td>
   <td>
-    <% if partition.lag_stored.to_i < 0 %>
-      <span class="badge bg-secondary" title="Not available until first offset commit">
-        N/A
-      </span>
-    <% else %>
-      <%= partition.lag_stored %>
-    <% end %>
+    <%== offset_with_label partition.lag_stored.to_i %>
   </td>
   <td>
     <span class="badge <%= lag_trend_bg(partition.lag_stored_d) %>">
@@ -20,16 +14,10 @@
     </span>
   </td>
   <td>
-    <%= partition.committed_offset %>
+    <%== offset_with_label partition.committed_offset.to_i %>
   </td>
   <td>
-    <% if partition.stored_offset.to_i < 0 %>
-      <span class="badge bg-secondary" title="Not available until first offset commit">
-        N/A
-      </span>
-    <% else %>
-      <%= partition.stored_offset %>
-    <% end %>
+    <%== offset_with_label partition.stored_offset.to_i %>
   </td>
   <td>
     <span class="badge <%= kafka_state_bg(partition.fetch_state) %> mt-1 mb-1">

data/lib/karafka/web/ui/pro/views/explorer/_detail.erb

@@ -9,6 +9,15 @@
       </td>
    </tr>
   <% end %>
+<% elsif v.is_a?(Time) %>
+  <tr>
+    <td>
+      <%= k %>
+    </td>
+    <td>
+      <%== time_with_label(v) %>
+    </td>
+  </tr>
 <% else %>
   <tr>
     <td>

data/lib/karafka/web/ui/pro/views/explorer/_message.erb

@@ -13,7 +13,7 @@
       <%= message.offset %>
     </td>
     <td>
-      <%== relative_time message.timestamp %>
+      <%== time_with_label(message.timestamp) %>
     </td>
     <td>
       <%= message.key %>

data/lib/karafka/web/ui/pro/views/health/_partition.erb

@@ -6,13 +6,7 @@
     <%= partition_id %>
   </td>
   <td>
-    <% if details[:lag_stored].to_i < 0 %>
-      <span class="badge bg-secondary" title="Not available until first offset commit">
-        N/A
-      </span>
-    <% else %>
-      <%= details[:lag_stored] %>
-    <% end %>
+    <%== offset_with_label details[:lag_stored].to_i %>
   </td>
   <td>
     <span class="badge <%= lag_trend_bg(details[:lag_stored_d]) %>">
@@ -23,13 +17,7 @@
     <%= details[:committed_offset] %>
   </td>
   <td>
-    <% if details[:stored_offset].to_i < 0 %>
-      <span class="badge bg-secondary" title="Not available until first offset commit">
-        N/A
-      </span>
-    <% else %>
-      <%= details[:stored_offset] %>
-    <% end %>
+    <%== offset_with_label details[:stored_offset].to_i %>
   </td>
   <td>
     <span class="badge <%= kafka_state_bg(details[:fetch_state]) %> mt-1 mb-1">

data/lib/karafka/web/ui/pro/views/jobs/_job.erb

@@ -19,13 +19,13 @@
     <code>#<%= job.type %></code>
   </td>
   <td>
-    <%= job.first_offset %>
+    <%== offset_with_label job.first_offset %>
   </td>
   <td>
-    <%= job.last_offset %>
+    <%== offset_with_label job.last_offset %>
   </td>
   <td>
-    <%= job.comitted_offset %>
+    <%== offset_with_label job.committed_offset %>
   </td>
   <td>
     <%== relative_time job.started_at %>

data/lib/karafka/web/version.rb

@@ -3,6 +3,6 @@
 module Karafka
   module Web
     # Current gem version
-    VERSION = '0.4.0'
+    VERSION = '0.5.0'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-web
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2023-04-07 00:00:00.000000000 Z
+date: 2023-04-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: erubi
@@ -57,7 +57,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.38
+        version: 2.0.40
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
@@ -67,7 +67,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.0.38
+        version: 2.0.40
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0