karafka-web 0.10.4 → 0.11.0.beta2

data/lib/karafka/web/pro/ui/views/topics/distributions/_edit_form.erb

@@ -0,0 +1,47 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<div class="card bg-base-100">
+  <div class="card-body">
+    <% form_path = topics_path(@topic.topic_name, 'distribution') %>
+    <form class="space-y-4" method="post" action="<%= form_path %>" data-turbo="false">
+      <%== csrf_tag(form_path, 'PUT') %>
+      <input type="hidden" name="_method" value="put">
+
+      <div class="flex items-center w-full mb-8">
+        <label class="w-1/4 text-gray-700">Current Partitions:</label>
+        <div class="w-3/4">
+          <code class="bg-base-200 px-2 py-1 rounded-sm"><%= @topic.partition_count %></code>
+        </div>
+      </div>
+
+      <div class="flex items-center w-full mb-8">
+        <label class="w-1/4 text-gray-700">New Partition Count:</label>
+        <div class="w-3/4">
+          <input
+            type="number"
+            name="partition_count"
+            class="block input input-bordered w-full"
+            placeholder="Enter new partition count"
+            value="<%= params.fetch(:partition_count, @topic.partition_count) %>"
+            min="<%= @topic.partition_count + 1 %>"
+            required
+          />
+          <div class="text-sm text-gray-500 mt-1">
+            Must be greater than current partition count (<%= @topic.partition_count %>)
+          </div>
+        </div>
+      </div>
+
+      <div class="flex justify-end gap-4">
+        <a href="<%= topics_path(@topic.topic_name, 'distribution') %>" class="btn btn-ghost">
+          Cancel
+        </a>
+        <button type="submit" class="btn btn-primary gap-2 btn-lockable confirm-action">
+          <%== icon(:plus) %>
+          Increase Partitions
+        </button>
+      </div>
+    </form>
+  </div>
+</div>

data/lib/karafka/web/pro/ui/views/topics/distributions/_edit_hints.erb

@@ -0,0 +1,15 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<div class="alert">
+  <div class="space-y-2">
+    <p>Before increasing partitions:</p>
+    <ul class="list-disc ml-6">
+      <li>Ensure all consumers support dynamic partition detection</li>
+      <li>Consider the impact on message ordering in your applications</li>
+      <li>Plan for temporary rebalancing as consumers detect the change</li>
+      <li>Monitor consumer lag during and after the operation</li>
+      <li>Consider increasing partitions during low-traffic periods</li>
+    </ul>
+  </div>
+</div>

data/lib/karafka/web/pro/ui/views/topics/distributions/_edit_warnings.erb

@@ -0,0 +1,14 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<% alert_box_warning('Partition Update Warning') do %>
+  <div class="space-y-4">
+    <ul class="list-disc ml-6">
+      <li>Increasing partitions is a one-way operation - partition count cannot be decreased later</li>
+      <li>Adding partitions affects message ordering and consistent hashing</li>
+      <li>Consumers will need to detect the partition count change and rebalance</li>
+      <li>Message distribution across partitions may become uneven until data rotates</li>
+      <li>Changes may take several minutes to be visible in the UI but will be applied immediately</li>
+    </ul>
+  </div>
+<% end %>

data/lib/karafka/web/pro/ui/views/topics/distributions/_empty_partitions.erb

@@ -0,0 +1,4 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<%== alert_info('Those partitions are empty and do not contain any data.') %>

data/lib/karafka/web/pro/ui/views/topics/{distribution → distributions}/_limited.erb

@@ -1,3 +1,6 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
 <div class="mb-10">
   <%==
     alert_info(

data/lib/karafka/web/pro/ui/views/topics/distributions/_partition.erb

@@ -0,0 +1,13 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<tr>
+  <td>
+    <a href="<%= explorer_topics_path(@topic.topic_name, partition.partition_id) %>">
+      <%= partition.partition_id %>
+    </a>
+  </td>
+  <td><%= partition.count %></td>
+  <td><%= partition.share %>%</td>
+  <td><%= partition.diff.round(2) %>%</td>
+</tr>

data/lib/karafka/web/pro/ui/views/topics/distributions/edit.erb

@@ -0,0 +1,16 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<% view_title "Topic #{@topic.topic_name} - Increase Partitions" %>
+
+<div class="space-y-4">
+  <%== partial 'topics/distributions/edit_warnings' %>
+
+  <% if @form_error %>
+    <%== partial 'shared/rdkafka_form_error_alert_box' %>
+  <% else %>
+    <%== partial 'topics/distributions/edit_hints' %>
+  <% end %>
+
+  <%== partial 'topics/distributions/edit_form' %>
+</div>

data/lib/karafka/web/pro/ui/views/topics/{distribution.erb → distributions/show.erb}

@@ -1,21 +1,25 @@
-<% view_title @topic.topic_name %>
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
 
-<%== partial 'topics/tabs' %>
+<% view_title "#{@topic.topic_name} Data Distribution" %>
+
+<%== partial 'topics/topics/tabs' %>
+  <%== partial 'topics/distributions/add_partitions_button' %>
 
 <% if @limited %>
-  <%== partial 'topics/distribution/limited' %>
+  <%== partial 'topics/distributions/limited' %>
 <% end %>
 
 <% if @aggregated.sum.zero? %>
-  <%== partial 'topics/distribution/empty_partitions' %>
+  <%== partial 'topics/distributions/empty_partitions' %>
 <% else %>
   <div class="col-span-12 mb-3">
     <% if @active_partitions.size >= 2 %>
-      <%== partial 'topics/distribution/chart' %>
+      <%== partial 'topics/distributions/chart' %>
     <% end %>
 
     <div id="refreshable" class="mt-4">
-      <%== partial 'topics/distribution/badges' %>
+      <%== partial 'topics/distributions/badges' %>
 
       <div class="data-table-wrapper">
         <table class="data-table">
@@ -31,7 +35,7 @@
             <%==
               each_partial(
                 @distribution,
-                'topics/distribution/partition'
+                'topics/distributions/partition'
               )
             %>
           </tbody>

data/lib/karafka/web/pro/ui/views/topics/offsets/_breadcrumbs.erb

@@ -0,0 +1,20 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<li>
+  <a href="<%= topics_path %>">
+    Topics
+  </a>
+</li>
+
+<li>
+  <a href="<%= topics_path(@topic.topic_name, 'config') %>">
+    <%= @topic.topic_name %>
+  </a>
+</li>
+
+<li>
+  <a href="<%= topics_path(@topic.topic_name, 'offsets') %>">
+    Offsets
+  </a>
+</li>

data/lib/karafka/web/pro/ui/views/topics/offsets/_partition.erb

@@ -0,0 +1,13 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<tr>
+  <td>
+    <a href="<%= explorer_topics_path(@topic[:topic_name], partition[:partition_id]) %>">
+      <%= partition[:partition_id] %>
+    </a>
+  </td>
+  <td><%= partition[:low] %></td>
+  <td><%= partition[:high] %></td>
+  <td><%= partition[:diff] %></td>
+</tr>

data/lib/karafka/web/pro/ui/views/topics/{offsets.erb → offsets/show.erb}

@@ -1,6 +1,9 @@
-<% view_title @topic.topic_name %>
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
 
-<%== partial 'topics/tabs' %>
+<% view_title "#{@topic.topic_name} Offsets" %>
+
+<%== partial 'topics/topics/tabs' %>
 
 <div class="data-table-wrapper">
   <table class="data-table">
@@ -16,7 +19,7 @@
       <%==
         each_partial(
           @offsets,
-          'topics/partition_offsets'
+          'topics/offsets/partition'
         )
       %>
     </tbody>

data/lib/karafka/web/pro/ui/views/topics/replications/_breadcrumbs.erb

@@ -0,0 +1,20 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<li>
+  <a href="<%= topics_path %>">
+    Topics
+  </a>
+</li>
+
+<li>
+  <a href="<%= topics_path(@topic.topic_name, 'config') %>">
+    <%= @topic.topic_name %>
+  </a>
+</li>
+
+<li>
+  <a href="<%= topics_path(@topic.topic_name, 'replication') %>">
+    Replication
+  </a>
+</li>

data/lib/karafka/web/pro/ui/views/topics/{_partition.erb → replications/_partition.erb}

@@ -1,6 +1,9 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
 <tr>
   <td>
-    <a href="<%= explorer_path(@topic[:topic_name], partition[:partition_id]) %>">
+    <a href="<%= explorer_topics_path(@topic[:topic_name], partition[:partition_id]) %>">
       <%= partition[:partition_id] %>
     </a>
   </td>

data/lib/karafka/web/pro/ui/views/topics/{replication.erb → replications/show.erb}

@@ -1,6 +1,9 @@
-<% view_title @topic.topic_name %>
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
 
-<%== partial 'topics/tabs' %>
+<% view_title "#{@topic.topic_name} Replication" %>
+
+<%== partial 'topics/topics/tabs' %>
 
 <div class="data-table-wrapper">
   <table class="data-table">
@@ -16,7 +19,7 @@
       <%==
         each_partial(
           @partitions,
-          'topics/partition'
+          'topics/replications/partition'
         )
       %>
     </tbody>

data/lib/karafka/web/pro/ui/views/topics/topics/_breadcrumbs.erb

@@ -0,0 +1,32 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<li>
+  <a href="<%= topics_path %>">
+    Topics
+  </a>
+</li>
+
+<% if action?(:new, :create) %>
+  <li>
+    <a href="<%= topics_path('new') %>">
+      Creating New Topic
+    </a>
+  </li>
+<% end %>
+
+<% if @topic %>
+  <li>
+    <a href="<%= topics_path(@topic.topic_name, 'config') %>">
+      <%= @topic.topic_name %>
+    </a>
+  </li>
+<% end %>
+
+<% if action?(:edit) %>
+  <li>
+    <a href="<%= topics_path(@topic.topic_name, 'delete') %>">
+      Removal Confirmation
+    </a>
+  </li>
+<% end %>

data/lib/karafka/web/pro/ui/views/topics/topics/_create_button.erb

@@ -0,0 +1,13 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<div class="col-span-12 flex justify-end space-x-2 mb-6">
+  <a
+    class="btn btn-primary btn-sm <%= 'btn-disabled' unless features.topics_management? %>"
+    href="<%= topics_path('new') %>"
+    title="Create new topic"
+  >
+    <%== icon(:plus) %>
+    Create Topic
+  </a>
+</div>

data/lib/karafka/web/pro/ui/views/topics/topics/_create_hints.erb

@@ -0,0 +1,15 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<% alert_box_info('Topic Creation Settings') do %>
+  <p>
+    You are about to create a new Kafka topic. Please note:
+  </p>
+  <ul class="list-disc ml-6 mt-2">
+    <li>Topic name cannot be changed after creation</li>
+    <li>Number of partitions can only be increased, never decreased</li>
+    <li>Additional settings can be configured from the topic configuration page</li>
+    <li>It may take Kafka up to few minutes to fully synchronize the new topic</li>
+    <li>Consumers may require additional time to discover the topic, depending on their metadata refresh frequency</li>
+  </ul>
+<% end %>

data/lib/karafka/web/pro/ui/views/topics/topics/_delete_form.erb

@@ -0,0 +1,36 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<div class="card bg-base-100">
+  <div class="card-body">
+    <div class="alert alert-error">
+      <div class="flex items-center gap-4">
+        <%== icon(:trash) %>
+        <span>
+          You are about to delete topic:
+          <span class="font-bold ml-1"><%= @topic.topic_name %></span>
+          with
+          <span class="font-bold ml-1"><%= @topic.partition_count %></span>
+          partitions
+        </span>
+      </div>
+    </div>
+
+    <% path = topics_path(@topic.topic_name) %>
+    <form action="<%= path %>" method="post" class="mt-6" data-turbo="false">
+      <%== csrf_tag(path, 'DELETE') %>
+      <input type="hidden" name="_method" value="delete">
+
+      <div class="flex justify-end gap-4">
+        <a href="<%= topics_path(@topic.topic_name, 'config') %>" class="btn btn-ghost">
+          Cancel
+        </a>
+
+        <button type="submit" class="btn btn-error gap-2 btn-lockable confirm-action">
+          <%== icon(:trash) %>
+          Delete Topic
+        </button>
+      </div>
+    </form>
+  </div>
+</div>

data/lib/karafka/web/pro/ui/views/topics/topics/_delete_hints.erb

@@ -0,0 +1,15 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<div class="alert">
+  <div class="space-y-2">
+    <p>Before proceeding, ensure that:</p>
+
+    <ul class="list-disc ml-6">
+      <li>All applications consuming from this topic have been properly shut down</li>
+      <li>All producers to this topic have been stopped</li>
+      <li>You have backed up any critical data if needed</li>
+      <li>You have notified relevant team members about this deletion</li>
+    </ul>
+  </div>
+</div>

data/lib/karafka/web/pro/ui/views/topics/topics/_delete_warning.erb

@@ -0,0 +1,13 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<% alert_box_warning('Topic Removal Warning') do %>
+  <div class="space-y-4">
+    <ul class="list-disc ml-6">
+      <li>All data in this topic will be permanently deleted and cannot be recovered</li>
+      <li>All consumers and producers for this topic will stop functioning</li>
+      <li>Applications dependent on this topic may experience errors or disruptions</li>
+      <li>Consumer group offsets associated with this topic will be lost</li>
+    </ul>
+  </div>
+<% end %>

data/lib/karafka/web/pro/ui/views/topics/topics/_new_form.erb

@@ -0,0 +1,80 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<div class="card bg-base-100 mt-4">
+  <div class="card-body border-error">
+    <form class="space-y-4 w-full" method="post" action="<%= topics_path %>" data-turbo="false">
+      <%== csrf_tag(topics_path) %>
+
+      <div class="flex items-center w-full mb-8">
+        <label class="w-1/4 text-gray-700">Topic Name:</label>
+        <div class="w-3/4">
+          <input
+            type="text"
+            name="topic_name"
+            class="block input input-bordered w-full"
+            placeholder="Enter topic name"
+            pattern="[A-Za-z0-9\-_.]+"
+            minlength="1"
+            maxlength="249"
+            value="<%= params.fetch(:topic_name, '') %>"
+            required
+          />
+          <div class="text-sm text-gray-500 mt-1">
+            Only alphanumeric characters, dots, underscores, and hyphens are allowed
+          </div>
+        </div>
+      </div>
+
+      <div class="flex items-center w-full mb-8">
+        <label class="w-1/4 text-gray-700">Number of Partitions:</label>
+        <div class="w-3/4">
+          <input
+            type="number"
+            min="1"
+            max="10000"
+            name="partitions_count"
+            class="block input input-bordered w-full"
+            placeholder="Enter number of partitions"
+            value="<%= params.fetch(:partitions_count, 5) %>"
+            required
+          />
+          <div class="text-sm text-gray-500 mt-1">
+            Minimum 1 partition, cannot be decreased after creation
+          </div>
+        </div>
+      </div>
+
+      <div class="flex items-center w-full">
+        <label class="w-1/4 text-gray-700">Replication Factor:</label>
+        <div class="w-3/4">
+          <input
+            type="number"
+            min="1"
+            max="100"
+            name="replication_factor"
+            class="block input input-bordered w-full"
+            placeholder="Enter replication factor"
+            value="<%= params.fetch(:replication_factor, 1) %>"
+            required
+          />
+          <div class="text-sm text-gray-500 mt-1">
+            Number of replicas for each partition (minimum 1, recommended 3 for production)
+          </div>
+        </div>
+      </div>
+
+      <div class="fieldset text-center mt-6">
+        <div class="flex gap-4 justify-end">
+          <a href="<%= topics_path %>" class="btn btn-ghost">
+            Cancel
+          </a>
+          <button type="submit" class="btn btn-primary gap-2 btn-lockable">
+            <%== icon(:plus) %>
+            Create Topic
+          </button>
+        </div>
+      </div>
+    </form>
+  </div>
+</div>

data/lib/karafka/web/pro/ui/views/topics/{_tabs.erb → topics/_tabs.erb}

@@ -1,28 +1,31 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
 <div class="tab-container-wrapper">
   <div class="tab-container">
     <a
-      href="<%= root_path('topics', @topic.topic_name, 'config') %>"
+      href="<%= topics_path(@topic.topic_name, 'config') %>"
       class="custom-tab <%= nav_class(end_with: 'config') %>"
     >
       Configuration
     </a>
 
     <a
-      href="<%= root_path('topics', @topic.topic_name, 'replication') %>"
+      href="<%= topics_path(@topic.topic_name, 'replication') %>"
       class="custom-tab <%= nav_class(end_with: 'replication') %>"
     >
       Replication
     </a>
 
     <a
-      href="<%= root_path('topics', @topic.topic_name, 'distribution') %>"
+      href="<%= topics_path(@topic.topic_name, 'distribution') %>"
       class="custom-tab <%= nav_class(end_with: 'distribution') %>"
     >
       Distribution
     </a>
 
     <a
-      href="<%= root_path('topics', @topic.topic_name, 'offsets') %>"
+      href="<%= topics_path(@topic.topic_name, 'offsets') %>"
       class="custom-tab <%= nav_class(end_with: 'offsets') %>"
     >
       Offsets

data/lib/karafka/web/pro/ui/views/topics/topics/_topic.erb

@@ -0,0 +1,12 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<div class="topic-tile">
+  <div class="topic-tile-body">
+    <p class="topic-tile-text">
+      <a href="<%= topics_path(topic.topic_name, 'config') %>" class="topic-tile-link">
+        <%= topic.topic_name %> / <%= topic.partition_count %>
+      </a>
+    </p>
+  </div>
+</div>

data/lib/karafka/web/pro/ui/views/topics/topics/edit.erb

@@ -0,0 +1,10 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<% view_title "Topic #{@topic.topic_name} Removal Confirmation" %>
+
+<div class="space-y-4">
+  <%== partial 'topics/topics/delete_warning' %>
+  <%== partial 'topics/topics/delete_hints' %>
+  <%== partial 'topics/topics/delete_form' %>
+</div>

data/lib/karafka/web/pro/ui/views/topics/topics/index.erb

@@ -0,0 +1,19 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<% view_title 'Topics' %>
+
+<%== partial 'topics/topics/create_button' %>
+
+<% if @topics.empty? %>
+  <%== partial 'explorer/explorer/no_topics' %>
+<% else %>
+  <div class="topic-tiles">
+    <%==
+      each_partial(
+        @topics,
+        'topics/topics/topic'
+      )
+    %>
+  </div>
+<% end %>

data/lib/karafka/web/pro/ui/views/topics/topics/new.erb

@@ -0,0 +1,12 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<% view_title "Creating New Topic" %>
+
+<% if @form_error %>
+  <%== partial 'shared/rdkafka_form_error_alert_box' %>
+<% else %>
+  <%== partial 'topics/topics/create_hints' %>
+<% end %>
+
+<%== partial 'topics/topics/new_form' %>

data/lib/karafka/web/processing/consumer.rb

@@ -29,14 +29,14 @@ module Karafka
 
               raise ::Karafka::Web::Errors::Processing::IncompatibleSchemaError
             # Older reports mean someone is in the middle of upgrade. Schema change related
-            # upgrades always should happen without a rolling-upgrade, hence we can reject those
-            # requests without significant or any impact on data quality but without having to
-            # worry about backwards compatibility. Errors are tracked independently, so it should
-            # not be a problem.
-            #
-            # In case user wants to do a rolling upgrade, the user docs state that this can happen
-            # and it is something user should be aware
+            # upgrades always should happen without a rolling-upgrade. For such, since we cannot
+            # reason about their statistics structure, we only track state, so we can provide
+            # basic upgrade reporting details in the status page. All other data is rejected and
+            # since in most cases this is intermediate due to rolling upgrades, this should not
+            # significantly impact the state tracking and processing.
             when :older
+              @state_aggregator.add_state(message.payload, message.offset)
+
               next
             else
               raise ::Karafka::Errors::UnsupportedCaseError