karafka-web 0.11.0.rc1 → 0.11.0.rc3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 323655d05b3e4b77ddce36c63cee362ef711c89256cbcb97db6ea57c8939a398
-  data.tar.gz: c26e39271b910a860a85a22296a85f37367706754448fd96379cf6d0e4d4f101
+  metadata.gz: 5263c3a83f8cbd944097fb7889c936836fc0dc09880f4d14e87783a1ea55142b
+  data.tar.gz: 5a4ae04fb60f88132ca5855879a689471969f9c1b3d957ef2d9798d112c81ef3
 SHA512:
-  metadata.gz: 9632693f31939e911fba49a2a2ecb6c85e8cd468a01daa275d910d2c23b3e9b39a1ebc83d99cf5ed552fe794d9c7c1fe05097e4a5fa718190c8c1979df0ce743
-  data.tar.gz: d90eeb89e20b214a94450896a8cc65b7fceb73e1b1f7c02aedddb0af3953b500c97b50e70bc993b9a54de074179650b680f96f4b36c0aefc9bc490784720e10d
+  metadata.gz: 7d22123886f177f58f20783e6abe09fd84fc8d7554ac6f47a7103176f2c48a700c7e9ec90f38e72859dc54e51c959c7486b2f7b43512f6e3f649ca87498c3bd8
+  data.tar.gz: 33eb094a22870bb238016b018aa986f9914523eda62587f9d3cfc462249a62bbd88e66d2cb605c2fb6309fc5c7d1a03412c7cc3b1db9a5560a6d9ea7e11aa205

data/CHANGELOG.md

@@ -27,6 +27,8 @@
 - [Enhancement] Use more performant Kafka API calls to describe topics.
 - [Enhancement] Inject `.action-NAME` and `.controller-NAME` body classes for usage with custom CSS and JS.
 - [Enhancement] Improve error handling in the commanding iterator listener (Pro).
+- [Enhancement] Introduce `trace_id` to the errors tracked for DLQ correlation (if in use) (Pro).
+- [Enhancement] Normalize how topics with partitions data is being displayed (`topic-[0,1,2]` etc).
 - [Change] Do not fully hide config-disabled features but make them disabled.
 - [Change] Remove per-consumer process duplicated details from Subscriptions and Jobs tabs.
 - [Change] Move to trusted-publishers and remove signing since no longer needed.
@@ -53,6 +55,10 @@
 - [Fix] Ensure that when flashes or alerts are visible, pages are not auto-refreshed (would cause them to dissapear).
 - [Fix] Time selector in the explorer does not disappear after clicking out.
 - [Fix] Tombstone message presentation epoch doesn't make sense.
+- [Fix] Fix incorrectly displayed "No jobs" alert info.
+- [Fix] Previous / next navigation in the explorer does not work when moving to transactional checkpoints.
+- [Fix] Errors explorer does not work with transactional produced data.
+- [Fix] Errors explorer in OSS does not have pagination.
 - [Maintenance] Require `karafka-core` `>= 2.4.8` and `karafka` `>= 2.4.16`.
 - [Maintenance] Update `AirDatepicker` to `3.6.0`.
 

data/Gemfile.lock

@@ -1,9 +1,9 @@
 PATH
   remote: .
   specs:
-    karafka-web (0.11.0.rc1)
+    karafka-web (0.11.0.rc3)
       erubi (~> 1.4)
-      karafka (>= 2.5.0.beta1, < 2.6.0)
+      karafka (>= 2.5.0.rc1, < 2.6.0)
       karafka-core (>= 2.5.0, < 2.6.0)
       roda (~> 3.68, >= 3.69)
       tilt (~> 2.0)
@@ -35,7 +35,7 @@ GEM
     erubi (1.13.1)
     et-orbi (1.2.11)
       tzinfo
-    factory_bot (6.5.3)
+    factory_bot (6.5.4)
       activesupport (>= 6.1.0)
     ffi (1.17.2)
     ffi (1.17.2-aarch64-linux-gnu)
@@ -53,13 +53,13 @@ GEM
       raabro (~> 1.4)
     i18n (1.14.7)
       concurrent-ruby (~> 1.0)
-    karafka (2.5.0.rc1)
+    karafka (2.5.0.rc2)
       base64 (~> 0.2)
-      karafka-core (>= 2.5.0, < 2.6.0)
+      karafka-core (>= 2.5.2, < 2.6.0)
       karafka-rdkafka (>= 0.19.5)
       waterdrop (>= 2.8.3, < 3.0.0)
       zeitwerk (~> 2.3)
-    karafka-core (2.5.1)
+    karafka-core (2.5.2)
       karafka-rdkafka (>= 0.19.2, < 0.21.0)
       logger (>= 1.6.0)
     karafka-rdkafka (0.19.5)

data/config/locales/slogans.yml

@@ -43,7 +43,7 @@ en:
     - "Are you concerned about managing large data flows? Virtual Partitions in Karafka Pro simplify data processing, offering unparalleled efficiency and scalability."
     - "Looking for flexible scheduling solutions? Karafka's Pro Periodic Jobs feature allows precise task timing and execution, enhancing your application's performance and reliability."
     - "Need advanced routing capabilities? Routing Patterns in Karafka Pro provide dynamic message routing, improving your system's adaptability and efficiency."
-    - "Are you Seeking to control your workload? With Rate Limiting in Karafka Pro, you can effectively manage and balance your system's load, ensuring optimal performance under any conditions."
+    - "Are you seeking to control your workload? With Rate Limiting in Karafka Pro, you can effectively manage and balance your system's load, ensuring optimal performance under any conditions."
     - "Need quick data access methods? The Iterator API in Karafka Pro provides efficient data retrieval methods."
     - "Want more nuanced error handling? Discover Granular Backoffs in Karafka Pro, which allows for detailed control over retry mechanisms and error recovery processes."
     - "Need a boost in data processing speed? Multiplexing in Karafka Pro enables parallel processing, significantly increasing throughput and reducing processing times."

data/docker-compose.yml

@@ -3,7 +3,7 @@ version: '2'
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.9.1
+    image: confluentinc/cp-kafka:8.0.0
 
     ports:
       - 9092:9092

data/karafka-web.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |spec|
   spec.licenses    = %w[LGPL-3.0-only Commercial]
 
   spec.add_dependency 'erubi', '~> 1.4'
-  spec.add_dependency 'karafka', '>= 2.5.0.beta1', '< 2.6.0'
+  spec.add_dependency 'karafka', '>= 2.5.0.rc1', '< 2.6.0'
   spec.add_dependency 'karafka-core', '>= 2.5.0', '< 2.6.0'
   spec.add_dependency 'roda', '~> 3.68', '>= 3.69'
   spec.add_dependency 'tilt', '~> 2.0'

data/lib/karafka/web/pro/ui/controllers/errors_controller.rb

@@ -46,7 +46,8 @@ module Karafka
                 previous_offset,
                 @params.current_offset,
                 next_offset,
-                @error_messages.map(&:offset)
+                # If message is an array, it means it's a compacted dummy offset representation
+                @error_messages.map { |message| message.is_a?(Array) ? message.last : message.offset }
               )
 
               render
@@ -59,13 +60,16 @@ module Karafka
             def show(partition_id, offset)
               @partition_id = partition_id
               @offset = offset
+
+              watermark_offsets = Models::WatermarkOffsets.find(errors_topic, partition_id)
+
               @error_message = Models::Message.find(
                 errors_topic,
-                @partition_id,
-                @offset
+                partition_id,
+                offset,
+                watermark_offsets: watermark_offsets
               )
 
-              watermark_offsets = Models::WatermarkOffsets.find(errors_topic, partition_id)
               paginate(offset, watermark_offsets.low, watermark_offsets.high)
 
               render

data/lib/karafka/web/pro/ui/controllers/explorer/explorer_controller.rb

@@ -91,20 +91,30 @@ module Karafka
                 @topic_id = topic_id
                 @partition_id = partition_id
                 @offset = offset
-                @message = Models::Message.find(@topic_id, @partition_id, @offset)
 
-                @safe_key = Web::Pro::Ui::Lib::SafeRunner.new { @message.key }.tap(&:call)
-                @safe_headers = Web::Pro::Ui::Lib::SafeRunner.new { @message.headers }.tap(&:call)
-                @safe_payload = Web::Pro::Ui::Lib::SafeRunner.new { @message.payload }.tap(&:call)
+                watermark_offsets = Models::WatermarkOffsets.find(topic_id, partition_id)
+
+                @message = Models::Message.find(
+                  @topic_id,
+                  @partition_id,
+                  @offset,
+                  watermark_offsets: watermark_offsets
+                )
+
+                # Only if it is not a compacted message we can deserialize stuff
+                if @message
+                  @safe_key = Web::Pro::Ui::Lib::SafeRunner.new { @message.key }.tap(&:call)
+                  @safe_headers = Web::Pro::Ui::Lib::SafeRunner.new { @message.headers }.tap(&:call)
+                  @safe_payload = Web::Pro::Ui::Lib::SafeRunner.new { @message.payload }.tap(&:call)
+                end
 
                 # This may be off for certain views like recent view where we are interested only
                 # in the most recent all the time. It does not make any sense to display pagination
                 # there
-                if paginate
-                  # We need watermark offsets to decide if we can paginate left and right
-                  watermark_offsets = Models::WatermarkOffsets.find(topic_id, partition_id)
-                  paginate(offset, watermark_offsets.low, watermark_offsets.high)
-                end
+                #
+                # We need watermark offsets to decide if we can paginate left and right
+                # We check that because it may have got here via the transactional flow
+                paginate(offset, watermark_offsets.low, watermark_offsets.high) if paginate
 
                 render
               end

data/lib/karafka/web/pro/ui/views/consumers/jobs/_job.erb

@@ -4,8 +4,7 @@
 <tr>
   <td>
     <span class="badge badge-secondary" title="Consumer group: <%= job.consumer_group %>">
-      <%= job.topic %>:
-      <%= job.partition %>
+      <%= topics_partition_identifier(job.topic, job.partition) %>
     </span>
   </td>
   <td>

data/lib/karafka/web/pro/ui/views/consumers/jobs/_no_jobs.erb

@@ -1,10 +1,6 @@
 <%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
 <%# See LICENSE for details. %>
 
-<div class="container mb-4">
-  <div class="row">
-    <div class="col-lg-12">
-      <%== alert_info("This process has no #{type} jobs at the moment.") %>
-    </div>
-  </div>
+<div class="col-lg-12">
+  <%== alert_info("This process has no #{type} jobs at the moment.") %>
 </div>

data/lib/karafka/web/pro/ui/views/consumers/partitions/offsets/edit.erb

@@ -1,7 +1,11 @@
 <%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
 <%# See LICENSE for details. %>
 
-<% view_title "#{@process.id} - Offset Adjustment - #{@subscription_group_id} - #{@topic}##{@partition_id}" %>
+<%
+  tpi = topics_partition_identifier(@topic, @partition_id)
+
+  view_title "#{@process.id} - Offset Adjustment - #{@subscription_group_id} - #{tpi}"
+%>
 
 <% if @process.status != 'running' %>
   <%== partial 'consumers/partitions/offsets/not_running_error' %>

data/lib/karafka/web/pro/ui/views/consumers/partitions/pauses/edit.erb

@@ -1,7 +1,11 @@
 <%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
 <%# See LICENSE for details. %>
 
-<% view_title "#{@process.id} - Pause Adjustment - #{@subscription_group_id} - #{@topic}##{@partition_id}" %>
+<%
+  tpi = topics_partition_identifier(@topic, @partition_id)
+
+  view_title "#{@process.id} - Pause Adjustment - #{@subscription_group_id} - #{tpi}"
+%>
 
 <% if @topic_lrj %>
   <%== partial 'consumers/partitions/pauses/lrj_not_manageable' %>

data/lib/karafka/web/pro/ui/views/consumers/partitions/pauses/new.erb

@@ -1,7 +1,11 @@
 <%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
 <%# See LICENSE for details. %>
 
-<% view_title "#{@process.id} - Pause Adjustment - #{@subscription_group_id} - #{@topic}##{@partition_id}" %>
+<%
+  tpi = topics_partition_identifier(@topic, @partition_id)
+
+  view_title "#{@process.id} - Pause Adjustment - #{@subscription_group_id} - #{tpi}"
+%>
 
 <% if @topic_lrj %>
   <%== partial 'consumers/partitions/pauses/lrj_not_manageable' %>

data/lib/karafka/web/pro/ui/views/errors/_breadcrumbs.erb

@@ -18,12 +18,7 @@
 <% if @offset %>
   <li>
     <a href="<%= root_path('errors', @partition_id, @offset) %>">
-      <%=
-        type = @error_message.payload[:type]
-        error_class = @error_message.payload[:error_class]
-
-        "#{type}: #{error_class}"
-      %>
+      <%= "Offset #{@offset}" %>
     </a>
   </li>
 <% end %>

data/lib/karafka/web/pro/ui/views/errors/_error.erb

@@ -10,7 +10,12 @@
     <td>
       <span class="badge badge-secondary">
         <% if error[:details].key?(:topic) %>
-          <%= error[:details][:topic] %>: <%= error[:details][:partition] %>
+          <%=
+            topic = error[:details][:topic]
+            partition = error[:details][:partition]
+
+            topics_partition_identifier(topic, partition)
+          %>
         <% else %>
           <%= error[:type] %>
         <% end %>

data/lib/karafka/web/pro/ui/views/errors/show.erb

@@ -1,42 +1,48 @@
 <%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
 <%# See LICENSE for details. %>
 
-<%
-  type = @error_message.payload[:type]
-  error_class = @error_message.payload[:error_class]
+<% if @error_message %>
+  <%
+    type = @error_message.payload[:type]
+    error_class = @error_message.payload[:error_class]
 
-  view_title("#{type}: #{error_class}")
-%>
+    view_title("#{type}: #{error_class}")
+  %>
 
-<div class="col-span-12 mb-3">
-  <h2 class="h2">
-    Metadata
-  </h2>
+  <div class="col-span-12 mb-3">
+    <h2 class="h2">
+      Metadata
+    </h2>
 
-  <div class="data-table-wrapper">
-    <table class="data-table">
-      <tbody>
-        <% @error_message.payload.each do |k, v| %>
-          <% next if k == :backtrace %>
-          <%==
-            partial(
-              'errors/detail',
-              locals: {
-                k: k,
-                v: v
-              }
-            )
-          %>
-        <% end %>
-      </tbody>
-    </table>
+    <div class="data-table-wrapper">
+      <table class="data-table">
+        <tbody>
+          <% @error_message.payload.each do |k, v| %>
+            <% next if k == :backtrace %>
+            <%==
+              partial(
+                'errors/detail',
+                locals: {
+                  k: k,
+                  v: v
+                }
+              )
+            %>
+          <% end %>
+        </tbody>
+      </table>
+    </div>
   </div>
-</div>
 
-<div class="col-span-12 mb-3">
-  <h2 class="h2">
-    Backtrace
-  </h2>
+  <div class="col-span-12 mb-3">
+    <h2 class="h2">
+      Backtrace
+    </h2>
 
-  <pre class="mb-5 p-0 border border-gray-300 text-sm"><code class="wrapped json p-0 m-0"><%= @error_message.payload[:backtrace] %></code></pre>
-</div>
+    <pre class="mb-5 p-0 border border-gray-300 text-sm"><code class="wrapped json p-0 m-0"><%= @error_message.payload[:backtrace] %></code></pre>
+  </div>
+<% else %>
+  <% view_title "Offset #{@offset}" %>
+
+  <%== partial 'shared/compacted_message_info' %>
+<% end %>

data/lib/karafka/web/pro/ui/views/explorer/explorer/show.erb

@@ -1,95 +1,100 @@
 <%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
 <%# See LICENSE for details. %>
 
-<% view_title "Message with offset #{@message.offset} from partition #{@message.partition}" %>
+<% if @message %>
+  <% view_title "Message with offset #{@offset} from partition #{@partition_id}" %>
 
-<%
-  republish_path = explorer_messages_path(@message.topic, @message.partition, @message.offset, 'republish')
-  surrounding_path = explorer_topics_path(@message.topic, @message.partition, @message.offset, 'surrounding')
-%>
+  <%
+    republish_path = explorer_messages_path(@message.topic, @message.partition, @message.offset, 'republish')
+    surrounding_path = explorer_topics_path(@message.topic, @message.partition, @message.offset, 'surrounding')
+  %>
+  <div class="col-span-12">
+    <section class="actions">
+      <% if @visibility_filter.republish?(@message) %>
+        <a
+          href="<%= explorer_messages_path(@message.topic, @message.partition, @message.offset, 'forward') %>"
+          class="btn-action"
+          title="Republish this message"
+        >
+          <%== icon(:arrow_uturn_right) %>
+        </a>
+      <% end %>
 
-<div class="col-span-12">
-  <section class="actions">
-    <% if @visibility_filter.republish?(@message) %>
       <a
-        href="<%= explorer_messages_path(@message.topic, @message.partition, @message.offset, 'forward') %>"
+        href="<%= surrounding_path %>"
         class="btn-action"
-        title="Republish this message"
+        title="Go to this message surrounding position"
       >
-        <%== icon(:arrow_uturn_right) %>
+        <%== icon(:arrows_right_left) %>
       </a>
-    <% end %>
 
-    <a
-      href="<%= surrounding_path %>"
-      class="btn-action"
-      title="Go to this message surrounding position"
-    >
-      <%== icon(:arrows_right_left) %>
-    </a>
+      <% if @visibility_filter.download?(@message) %>
+        <a
+          href="<%= explorer_messages_path(@message.topic, @message.partition, @message.offset, 'download') %>"
+          class="btn-action"
+          title="Download raw payload"
+          data-turbo="false"
+        >
+          <%== icon(:arrow_down_tray) %>
+        </a>
+      <% end %>
 
-    <% if @visibility_filter.download?(@message) %>
-      <a
-        href="<%= explorer_messages_path(@message.topic, @message.partition, @message.offset, 'download') %>"
-        class="btn-action"
-        title="Download raw payload"
-        data-turbo="false"
+      <% if @visibility_filter.export?(@message) && @safe_payload.success? %>
+        <a
+          href="<%= explorer_messages_path(@message.topic, @message.partition, @message.offset, 'export') %>"
+          class="btn-action"
+          title="Export as JSON"
+          data-turbo="false"
+        >
+          <%== icon(:arrow_down_on_square) %>
+        </a>
+      <% end %>
+
+      <button
+        class="btn-action btn-toggle"
+        title="Show the message metadata details"
+        id="metadata-button"
+        data-toggle-target="metadata-details"
       >
-        <%== icon(:arrow_down_tray) %>
-      </a>
-    <% end %>
+        <%== icon(:circle_stack) %>
+      </button>
 
-    <% if @visibility_filter.export?(@message) && @safe_payload.success? %>
-      <a
-        href="<%= explorer_messages_path(@message.topic, @message.partition, @message.offset, 'export') %>"
-        class="btn-action"
-        title="Export as JSON"
-        data-turbo="false"
+      <button
+        class="btn-action btn-toggle"
+        title="Show the message utilization details"
+        id="utilization-button"
+        data-toggle-target="utilization-details"
       >
-        <%== icon(:arrow_down_on_square) %>
-      </a>
-    <% end %>
+        <%== icon(:cpu) %>
+      </button>
+    </section>
+  </div>
 
-    <button
-      class="btn-action btn-toggle"
-      title="Show the message metadata details"
-      id="metadata-button"
-      data-toggle-target="metadata-details"
-    >
-      <%== icon(:circle_stack) %>
-    </button>
+  <div class="col-span-12 mb-6">
+    <div id="utilization-details" class="hidden">
+      <h2 class="h2">
+        Utilization
+      </h2>
 
-    <button
-      class="btn-action btn-toggle"
-      title="Show the message utilization details"
-      id="utilization-button"
-      data-toggle-target="utilization-details"
-    >
-      <%== icon(:cpu) %>
-    </button>
-  </section>
-</div>
+      <%== partial 'explorer/explorer/message/resources_utilization' %>
+    </div>
 
-<div class="col-span-12 mb-6">
-  <div id="utilization-details" class="hidden">
-    <h2 class="h2">
-      Utilization
-    </h2>
+    <div id="metadata-details">
+      <h2 class="h2">
+        Metadata
+      </h2>
 
-    <%== partial 'explorer/explorer/message/resources_utilization' %>
-  </div>
+      <%== partial 'explorer/explorer/message/metadata' %>
+    </div>
 
-  <div id="metadata-details">
     <h2 class="h2">
-      Metadata
+      Payload
     </h2>
 
-    <%== partial 'explorer/explorer/message/metadata' %>
+    <%== partial 'explorer/explorer/message/payload' %>
   </div>
+<% else %>
+  <% view_title "Message with offset #{@offset} from partition #{@partition_id} (No Data)" %>
 
-  <h2 class="h2">
-    Payload
-  </h2>
-
-  <%== partial 'explorer/explorer/message/payload' %>
-</div>
+  <%== partial 'shared/compacted_message_info' %>
+<% end %>

data/lib/karafka/web/pro/ui/views/jobs/_job.erb

@@ -9,8 +9,7 @@
   </td>
   <td>
     <span class="badge badge-secondary" title="Consumer group: <%= job.consumer_group %>">
-      <%= job.topic %>:
-      <%= job.partition %>
+      <%= topics_partition_identifier(job.topic, job.partition) %>
     </span>
   </td>
   <td>

data/lib/karafka/web/tracking/consumers/listeners/errors.rb

@@ -85,6 +85,7 @@ module Karafka
                 # with -1001, which is "N/A" offset position for all the offsets here
                 committed_offset: (consumer.coordinator.seek_offset || -1_000) - 1,
                 consumer: consumer.class.to_s,
+                trace_id: Karafka.pro? ? consumer.errors_tracker.trace_id : nil,
                 tags: consumer.tags
               }
             end

data/lib/karafka/web/ui/base.rb

@@ -7,6 +7,8 @@ module Karafka
       class Base < Roda
         include Helpers::PathsHelper
         include Helpers::ApplicationHelper
+        include Helpers::TimeHelper
+        include Helpers::TopicsHelper
         include Helpers::TailwindHelper
 
         # Details that need to be evaluated in the context of OSS or Pro web UI.

data/lib/karafka/web/ui/controllers/errors_controller.rb

@@ -17,7 +17,8 @@ module Karafka
               previous_offset,
               @params.current_offset,
               next_offset,
-              @error_messages.map(&:offset)
+              # If message is an array, it means it's a compacted dummy offset representation
+              @error_messages.map { |message| message.is_a?(Array) ? message.last : message.offset }
             )
 
             render
@@ -25,12 +26,20 @@ module Karafka
 
           # @param offset [Integer] given error message offset
           def show(offset)
+            @partition_id = 0
+            @offset = offset
+
+            watermark_offsets = Models::WatermarkOffsets.find(errors_topic, @partition_id)
+
             @error_message = Models::Message.find(
               errors_topic,
-              0,
-              offset
+              @partition_id,
+              offset,
+              watermark_offsets: watermark_offsets
             )
 
+            paginate(offset, watermark_offsets.low, watermark_offsets.high)
+
             render
           end