karafka-web 0.10.3 → 0.11.0.beta1

data/lib/karafka/web/pro/ui/views/topics/{distribution.erb → distributions/show.erb}

@@ -1,21 +1,25 @@
-<% view_title @topic.topic_name %>
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
 
-<%== partial 'topics/tabs' %>
+<% view_title "#{@topic.topic_name} Data Distribution" %>
+
+<%== partial 'topics/topics/tabs' %>
+  <%== partial 'topics/distributions/add_partitions_button' %>
 
 <% if @limited %>
-  <%== partial 'topics/distribution/limited' %>
+  <%== partial 'topics/distributions/limited' %>
 <% end %>
 
 <% if @aggregated.sum.zero? %>
-  <%== partial 'topics/distribution/empty_partitions' %>
+  <%== partial 'topics/distributions/empty_partitions' %>
 <% else %>
   <div class="col-span-12 mb-3">
     <% if @active_partitions.size >= 2 %>
-      <%== partial 'topics/distribution/chart' %>
+      <%== partial 'topics/distributions/chart' %>
     <% end %>
 
     <div id="refreshable" class="mt-4">
-      <%== partial 'topics/distribution/badges' %>
+      <%== partial 'topics/distributions/badges' %>
 
       <div class="data-table-wrapper">
         <table class="data-table">
@@ -31,7 +35,7 @@
             <%==
               each_partial(
                 @distribution,
-                'topics/distribution/partition'
+                'topics/distributions/partition'
               )
             %>
           </tbody>

data/lib/karafka/web/pro/ui/views/topics/offsets/_breadcrumbs.erb

@@ -0,0 +1,20 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<li>
+  <a href="<%= topics_path %>">
+    Topics
+  </a>
+</li>
+
+<li>
+  <a href="<%= topics_path(@topic.topic_name, 'config') %>">
+    <%= @topic.topic_name %>
+  </a>
+</li>
+
+<li>
+  <a href="<%= topics_path(@topic.topic_name, 'offsets') %>">
+    Offsets
+  </a>
+</li>

data/lib/karafka/web/pro/ui/views/topics/offsets/_partition.erb

@@ -0,0 +1,13 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<tr>
+  <td>
+    <a href="<%= explorer_topics_path(@topic[:topic_name], partition[:partition_id]) %>">
+      <%= partition[:partition_id] %>
+    </a>
+  </td>
+  <td><%= partition[:low] %></td>
+  <td><%= partition[:high] %></td>
+  <td><%= partition[:diff] %></td>
+</tr>

data/lib/karafka/web/pro/ui/views/topics/{offsets.erb → offsets/show.erb}

@@ -1,6 +1,9 @@
-<% view_title @topic.topic_name %>
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
 
-<%== partial 'topics/tabs' %>
+<% view_title "#{@topic.topic_name} Offsets" %>
+
+<%== partial 'topics/topics/tabs' %>
 
 <div class="data-table-wrapper">
   <table class="data-table">
@@ -16,7 +19,7 @@
       <%==
         each_partial(
           @offsets,
-          'topics/partition_offsets'
+          'topics/offsets/partition'
         )
       %>
     </tbody>

data/lib/karafka/web/pro/ui/views/topics/replications/_breadcrumbs.erb

@@ -0,0 +1,20 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<li>
+  <a href="<%= topics_path %>">
+    Topics
+  </a>
+</li>
+
+<li>
+  <a href="<%= topics_path(@topic.topic_name, 'config') %>">
+    <%= @topic.topic_name %>
+  </a>
+</li>
+
+<li>
+  <a href="<%= topics_path(@topic.topic_name, 'replication') %>">
+    Replication
+  </a>
+</li>

data/lib/karafka/web/pro/ui/views/topics/{_partition.erb → replications/_partition.erb}

@@ -1,6 +1,9 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
 <tr>
   <td>
-    <a href="<%= explorer_path(@topic[:topic_name], partition[:partition_id]) %>">
+    <a href="<%= explorer_topics_path(@topic[:topic_name], partition[:partition_id]) %>">
       <%= partition[:partition_id] %>
     </a>
   </td>

data/lib/karafka/web/pro/ui/views/topics/{replication.erb → replications/show.erb}

@@ -1,6 +1,9 @@
-<% view_title @topic.topic_name %>
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
 
-<%== partial 'topics/tabs' %>
+<% view_title "#{@topic.topic_name} Replication" %>
+
+<%== partial 'topics/topics/tabs' %>
 
 <div class="data-table-wrapper">
   <table class="data-table">
@@ -16,7 +19,7 @@
       <%==
         each_partial(
           @partitions,
-          'topics/partition'
+          'topics/replications/partition'
         )
       %>
     </tbody>

data/lib/karafka/web/pro/ui/views/topics/topics/_breadcrumbs.erb

@@ -0,0 +1,32 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<li>
+  <a href="<%= topics_path %>">
+    Topics
+  </a>
+</li>
+
+<% if action?(:new, :create) %>
+  <li>
+    <a href="<%= topics_path('new') %>">
+      Creating New Topic
+    </a>
+  </li>
+<% end %>
+
+<% if @topic %>
+  <li>
+    <a href="<%= topics_path(@topic.topic_name, 'config') %>">
+      <%= @topic.topic_name %>
+    </a>
+  </li>
+<% end %>
+
+<% if action?(:edit) %>
+  <li>
+    <a href="<%= topics_path(@topic.topic_name, 'delete') %>">
+      Removal Confirmation
+    </a>
+  </li>
+<% end %>

data/lib/karafka/web/pro/ui/views/topics/topics/_create_button.erb

@@ -0,0 +1,13 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<div class="col-span-12 flex justify-end space-x-2 mb-6">
+  <a
+    class="btn btn-primary btn-sm <%= 'btn-disabled' unless features.topics_management? %>"
+    href="<%= topics_path('new') %>"
+    title="Create new topic"
+  >
+    <%== icon(:plus) %>
+    Create Topic
+  </a>
+</div>

data/lib/karafka/web/pro/ui/views/topics/topics/_create_hints.erb

@@ -0,0 +1,15 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<% alert_box_info('Topic Creation Settings') do %>
+  <p>
+    You are about to create a new Kafka topic. Please note:
+  </p>
+  <ul class="list-disc ml-6 mt-2">
+    <li>Topic name cannot be changed after creation</li>
+    <li>Number of partitions can only be increased, never decreased</li>
+    <li>Additional settings can be configured from the topic configuration page</li>
+    <li>It may take Kafka up to few minutes to fully synchronize the new topic</li>
+    <li>Consumers may require additional time to discover the topic, depending on their metadata refresh frequency</li>
+  </ul>
+<% end %>

data/lib/karafka/web/pro/ui/views/topics/topics/_delete_form.erb

@@ -0,0 +1,36 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<div class="card bg-base-100">
+  <div class="card-body">
+    <div class="alert alert-error">
+      <div class="flex items-center gap-4">
+        <%== icon(:trash) %>
+        <span>
+          You are about to delete topic:
+          <span class="font-bold ml-1"><%= @topic.topic_name %></span>
+          with
+          <span class="font-bold ml-1"><%= @topic.partition_count %></span>
+          partitions
+        </span>
+      </div>
+    </div>
+
+    <% path = topics_path(@topic.topic_name) %>
+    <form action="<%= path %>" method="post" class="mt-6" data-turbo="false">
+      <%== csrf_tag(path, 'DELETE') %>
+      <input type="hidden" name="_method" value="delete">
+
+      <div class="flex justify-end gap-4">
+        <a href="<%= topics_path(@topic.topic_name, 'config') %>" class="btn btn-ghost">
+          Cancel
+        </a>
+
+        <button type="submit" class="btn btn-error gap-2 btn-lockable confirm-action">
+          <%== icon(:trash) %>
+          Delete Topic
+        </button>
+      </div>
+    </form>
+  </div>
+</div>

data/lib/karafka/web/pro/ui/views/topics/topics/_delete_hints.erb

@@ -0,0 +1,15 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<div class="alert">
+  <div class="space-y-2">
+    <p>Before proceeding, ensure that:</p>
+
+    <ul class="list-disc ml-6">
+      <li>All applications consuming from this topic have been properly shut down</li>
+      <li>All producers to this topic have been stopped</li>
+      <li>You have backed up any critical data if needed</li>
+      <li>You have notified relevant team members about this deletion</li>
+    </ul>
+  </div>
+</div>

data/lib/karafka/web/pro/ui/views/topics/topics/_delete_warning.erb

@@ -0,0 +1,13 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<% alert_box_warning('Topic Removal Warning') do %>
+  <div class="space-y-4">
+    <ul class="list-disc ml-6">
+      <li>All data in this topic will be permanently deleted and cannot be recovered</li>
+      <li>All consumers and producers for this topic will stop functioning</li>
+      <li>Applications dependent on this topic may experience errors or disruptions</li>
+      <li>Consumer group offsets associated with this topic will be lost</li>
+    </ul>
+  </div>
+<% end %>

data/lib/karafka/web/pro/ui/views/topics/topics/_new_form.erb

@@ -0,0 +1,80 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<div class="card bg-base-100 mt-4">
+  <div class="card-body border-error">
+    <form class="space-y-4 w-full" method="post" action="<%= topics_path %>" data-turbo="false">
+      <%== csrf_tag(topics_path) %>
+
+      <div class="flex items-center w-full mb-8">
+        <label class="w-1/4 text-gray-700">Topic Name:</label>
+        <div class="w-3/4">
+          <input
+            type="text"
+            name="topic_name"
+            class="block input input-bordered w-full"
+            placeholder="Enter topic name"
+            pattern="[A-Za-z0-9\-_.]+"
+            minlength="1"
+            maxlength="249"
+            value="<%= params.fetch(:topic_name, '') %>"
+            required
+          />
+          <div class="text-sm text-gray-500 mt-1">
+            Only alphanumeric characters, dots, underscores, and hyphens are allowed
+          </div>
+        </div>
+      </div>
+
+      <div class="flex items-center w-full mb-8">
+        <label class="w-1/4 text-gray-700">Number of Partitions:</label>
+        <div class="w-3/4">
+          <input
+            type="number"
+            min="1"
+            max="10000"
+            name="partitions_count"
+            class="block input input-bordered w-full"
+            placeholder="Enter number of partitions"
+            value="<%= params.fetch(:partitions_count, 5) %>"
+            required
+          />
+          <div class="text-sm text-gray-500 mt-1">
+            Minimum 1 partition, cannot be decreased after creation
+          </div>
+        </div>
+      </div>
+
+      <div class="flex items-center w-full">
+        <label class="w-1/4 text-gray-700">Replication Factor:</label>
+        <div class="w-3/4">
+          <input
+            type="number"
+            min="1"
+            max="100"
+            name="replication_factor"
+            class="block input input-bordered w-full"
+            placeholder="Enter replication factor"
+            value="<%= params.fetch(:replication_factor, 1) %>"
+            required
+          />
+          <div class="text-sm text-gray-500 mt-1">
+            Number of replicas for each partition (minimum 1, recommended 3 for production)
+          </div>
+        </div>
+      </div>
+
+      <div class="fieldset text-center mt-6">
+        <div class="flex gap-4 justify-end">
+          <a href="<%= topics_path %>" class="btn btn-ghost">
+            Cancel
+          </a>
+          <button type="submit" class="btn btn-primary gap-2 btn-lockable">
+            <%== icon(:plus) %>
+            Create Topic
+          </button>
+        </div>
+      </div>
+    </form>
+  </div>
+</div>

data/lib/karafka/web/pro/ui/views/topics/{_tabs.erb → topics/_tabs.erb}

@@ -1,28 +1,31 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
 <div class="tab-container-wrapper">
   <div class="tab-container">
     <a
-      href="<%= root_path('topics', @topic.topic_name, 'config') %>"
+      href="<%= topics_path(@topic.topic_name, 'config') %>"
       class="custom-tab <%= nav_class(end_with: 'config') %>"
     >
       Configuration
     </a>
 
     <a
-      href="<%= root_path('topics', @topic.topic_name, 'replication') %>"
+      href="<%= topics_path(@topic.topic_name, 'replication') %>"
       class="custom-tab <%= nav_class(end_with: 'replication') %>"
     >
       Replication
     </a>
 
     <a
-      href="<%= root_path('topics', @topic.topic_name, 'distribution') %>"
+      href="<%= topics_path(@topic.topic_name, 'distribution') %>"
       class="custom-tab <%= nav_class(end_with: 'distribution') %>"
     >
       Distribution
     </a>
 
     <a
-      href="<%= root_path('topics', @topic.topic_name, 'offsets') %>"
+      href="<%= topics_path(@topic.topic_name, 'offsets') %>"
       class="custom-tab <%= nav_class(end_with: 'offsets') %>"
     >
       Offsets

data/lib/karafka/web/pro/ui/views/topics/topics/_topic.erb

@@ -0,0 +1,12 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<div class="topic-tile">
+  <div class="topic-tile-body">
+    <p class="topic-tile-text">
+      <a href="<%= topics_path(topic.topic_name, 'config') %>" class="topic-tile-link">
+        <%= topic.topic_name %> / <%= topic.partition_count %>
+      </a>
+    </p>
+  </div>
+</div>

data/lib/karafka/web/pro/ui/views/topics/topics/edit.erb

@@ -0,0 +1,10 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<% view_title "Topic #{@topic.topic_name} Removal Confirmation" %>
+
+<div class="space-y-4">
+  <%== partial 'topics/topics/delete_warning' %>
+  <%== partial 'topics/topics/delete_hints' %>
+  <%== partial 'topics/topics/delete_form' %>
+</div>

data/lib/karafka/web/pro/ui/views/topics/topics/index.erb

@@ -0,0 +1,19 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<% view_title 'Topics' %>
+
+<%== partial 'topics/topics/create_button' %>
+
+<% if @topics.empty? %>
+  <%== partial 'explorer/explorer/no_topics' %>
+<% else %>
+  <div class="topic-tiles">
+    <%==
+      each_partial(
+        @topics,
+        'topics/topics/topic'
+      )
+    %>
+  </div>
+<% end %>

data/lib/karafka/web/pro/ui/views/topics/topics/new.erb

@@ -0,0 +1,12 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<% view_title "Creating New Topic" %>
+
+<% if @form_error %>
+  <%== partial 'shared/rdkafka_form_error_alert_box' %>
+<% else %>
+  <%== partial 'topics/topics/create_hints' %>
+<% end %>
+
+<%== partial 'topics/topics/new_form' %>

data/lib/karafka/web/processing/consumers/metrics.rb

@@ -12,7 +12,7 @@ module Karafka
             # @return [Hash] latest (current) aggregated metrics state
             def current!
               metrics_message = ::Karafka::Admin.read_topic(
-                Karafka::Web.config.topics.consumers.metrics,
+                Karafka::Web.config.topics.consumers.metrics.name,
                 0,
                 # We need to take more in case there would be transactions running.
                 # In theory we could take two but this compensates for any involuntary

data/lib/karafka/web/processing/consumers/state.rb

@@ -12,7 +12,7 @@ module Karafka
             # @return [Hash] last (current) aggregated processes state
             def current!
               state_message = ::Karafka::Admin.read_topic(
-                Karafka::Web.config.topics.consumers.states,
+                Karafka::Web.config.topics.consumers.states.name,
                 0,
                 # We need to take more in case there would be transactions running.
                 # In theory we could take two but this compensates for any involuntary

data/lib/karafka/web/processing/publisher.rb

@@ -36,17 +36,17 @@ module Karafka
           def prepare_data(consumers_state, consumers_metrics)
             [
               {
-                topic: Karafka::Web.config.topics.consumers.states,
+                topic: Karafka::Web.config.topics.consumers.states.name,
                 payload: Zlib::Deflate.deflate(consumers_state.to_json),
                 # This will ensure that the consumer states are compacted
-                key: Karafka::Web.config.topics.consumers.states,
+                key: Karafka::Web.config.topics.consumers.states.name,
                 partition: 0,
                 headers: { 'zlib' => 'true' }
               },
               {
-                topic: Karafka::Web.config.topics.consumers.metrics,
+                topic: Karafka::Web.config.topics.consumers.metrics.name,
                 payload: Zlib::Deflate.deflate(consumers_metrics.to_json),
-                key: Karafka::Web.config.topics.consumers.metrics,
+                key: Karafka::Web.config.topics.consumers.metrics.name,
                 partition: 0,
                 headers: { 'zlib' => 'true' }
               }

data/lib/karafka/web/tracking/consumers/contracts/partition.rb

@@ -28,6 +28,7 @@ module Karafka
             required(:ls_offset) { |val| val.is_a?(Integer) }
             required(:ls_offset_d) { |val| val.is_a?(Integer) }
             required(:ls_offset_fd) { |val| val.is_a?(Integer) && val >= 0 }
+            required(:transactional) { |val| [true, false].include?(val) }
           end
         end
       end

data/lib/karafka/web/tracking/consumers/listeners/errors.rb

@@ -18,17 +18,24 @@ module Karafka
             #
             # @param event [Karafka::Core::Monitoring::Event]
             def on_error_occurred(event)
-              track do |sampler|
-                # Collect extra info if it was a consumer related error.
-                # Those come from user code
-                details = if event[:caller].is_a?(Karafka::BaseConsumer)
-                            extract_consumer_info(event[:caller])
-                          else
-                            {}
-                          end
+              caller_ref = event[:caller]
+
+              # Collect extra info if it was a consumer related error.
+              # Those come from user code
+              details = case caller_ref
+                        when Karafka::BaseConsumer
+                          extract_consumer_info(caller_ref)
+                        when Karafka::Connection::Client
+                          extract_client_info(caller_ref)
+                        when Karafka::Connection::Listener
+                          extract_listener_info(caller_ref)
+                        else
+                          {}
+                        end
 
-                error_class, error_message, backtrace = extract_error_info(event[:error])
+              error_class, error_message, backtrace = extract_error_info(event[:error])
 
+              track do |sampler|
                 sampler.errors << {
                   schema_version: SCHEMA_VERSION,
                   type: event[:type],
@@ -70,6 +77,7 @@ module Karafka
               {
                 topic: consumer.topic.name,
                 consumer_group: consumer.topic.consumer_group.id,
+                subscription_group: consumer.topic.subscription_group.id,
                 partition: consumer.partition,
                 first_offset: consumer.messages.metadata.first_offset,
                 last_offset: consumer.messages.metadata.last_offset,
@@ -80,6 +88,27 @@ module Karafka
                 tags: consumer.tags
               }
             end
+
+            # @param client [::Karafka::Connection::Client]
+            # @return [Hash] hash with client specific info for details of error
+            def extract_client_info(client)
+              {
+                consumer_group: client.subscription_group.consumer_group.id,
+                subscription_group: client.subscription_group.id,
+                name: client.name,
+                id: client.id
+              }
+            end
+
+            # @param listener [::Karafka::Connection::Listener]
+            # @return [Hash] hash with listener specific info for details of error
+            def extract_listener_info(listener)
+              {
+                consumer_group: listener.subscription_group.consumer_group.id,
+                subscription_group: listener.subscription_group.id,
+                id: listener.id
+              }
+            end
           end
         end
       end

data/lib/karafka/web/tracking/consumers/listeners/pausing.rb

@@ -8,10 +8,10 @@ module Karafka
           # Tracks pausing and un-pausing of topics partitions for both user requested and
           # automatic events.
           class Pausing < Base
-            # Indicate pause
+            # Tracks the pause start
             #
             # @param event [Karafka::Core::Monitoring::Event]
-            def on_consumer_consuming_pause(event)
+            def on_client_pause(event)
               track do |sampler|
                 sampler.pauses[pause_id(event)] = {
                   timeout: event[:timeout],

data/lib/karafka/web/tracking/consumers/listeners/transactions.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Consumers
+        module Listeners
+          # Tracks data related to transactions
+          # seek offsets are needed because when consumer offsets are committed in transactions,
+          # librdkafka does not publish the lags in a regular way (they are set to -1) and we need
+          # to compute them via enrichment of information.
+          class Transactions < Base
+            # Tracking of things needed to support transactional consumers post successful
+            # transaction.
+            #
+            # @param event [Karafka::Core::Monitoring::Event]
+            def on_consumer_consuming_transaction(event)
+              consumer = event[:caller]
+              sg_id = consumer.topic.subscription_group.id
+              topic_name = consumer.topic.name
+              # We store it as a string because librdkafka also does that and its easier to align
+              # without casting it later
+              partition_id = consumer.partition
+
+              track do |sampler|
+                break unless sampler.subscription_groups.key?(sg_id)
+
+                seek_offset = consumer.coordinator.seek_offset
+
+                break if seek_offset.nil?
+
+                topics_scope = sampler.subscription_groups[sg_id][:topics]
+                p_scope = topics_scope[topic_name][partition_id]
+
+                p_scope[:transactional] = true
+                p_scope[:seek_offset] = seek_offset
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end