karafka 2.4.18 → 2.5.0.beta2

data/lib/karafka/connection/listener.rb

@@ -12,6 +12,13 @@ module Karafka
     class Listener
       include Helpers::Async
 
+      include Helpers::ConfigImporter.new(
+        jobs_builder: %i[internal processing jobs_builder],
+        partitioner_class: %i[internal processing partitioner_class],
+        reset_backoff: %i[internal connection reset_backoff],
+        listener_thread_priority: %i[internal connection listener_thread_priority]
+      )
+
       # Can be useful for logging
       # @return [String] id of this listener
       attr_reader :id
@@ -19,6 +26,11 @@ module Karafka
       # @return [Karafka::Routing::SubscriptionGroup] subscription group that this listener handles
       attr_reader :subscription_group
 
+      # @return [Processing::CoordinatorsBuffer] coordinator buffers that can be used directly in
+      #   advanced cases of changes to the polling flow (like triggered seek back without messages
+      #   ahead in the topic)
+      attr_reader :coordinators
+
       # How long to wait in the initial events poll. Increases chances of having the initial events
       # immediately available
       INITIAL_EVENTS_POLL_TIMEOUT = 100
@@ -30,16 +42,13 @@ module Karafka
       # @param scheduler [Karafka::Processing::Scheduler] scheduler we want to use
       # @return [Karafka::Connection::Listener] listener instance
       def initialize(subscription_group, jobs_queue, scheduler)
-        proc_config = ::Karafka::App.config.internal.processing
-
         @id = SecureRandom.hex(6)
         @subscription_group = subscription_group
         @jobs_queue = jobs_queue
         @coordinators = Processing::CoordinatorsBuffer.new(subscription_group.topics)
         @client = Client.new(@subscription_group, -> { running? })
         @executors = Processing::ExecutorsBuffer.new(@client, subscription_group)
-        @jobs_builder = proc_config.jobs_builder
-        @partitioner = proc_config.partitioner_class.new(subscription_group)
+        @partitioner = partitioner_class.new(subscription_group)
         @scheduler = scheduler
         @events_poller = Helpers::IntervalRunner.new { @client.events_poll }
         # We keep one buffer for messages to preserve memory and not allocate extra objects
@@ -111,7 +120,10 @@ module Karafka
 
         @status.start!
 
-        async_call("karafka.listener##{@subscription_group.id}")
+        async_call(
+          "karafka.listener##{@subscription_group.id}",
+          listener_thread_priority
+        )
       end
 
       # Stops the jobs queue, triggers shutdown on all the executors (sync), commits offsets and
@@ -254,7 +266,7 @@ module Karafka
         reset
 
         # Ruby sleep is in seconds
-        sleep_time = ::Karafka::App.config.internal.connection.reset_backoff / 10_000.0
+        sleep_time = reset_backoff / 10_000.0
         sleep(sleep_time) && retry
       end
 
@@ -294,7 +306,7 @@ module Karafka
             # here. In cases like this, we do not run a revocation job
             @executors.find_all(topic, partition).each do |executor|
               executor.coordinator.increment(:revoked)
-              jobs << @jobs_builder.revoked(executor)
+              jobs << jobs_builder.revoked(executor)
             end
 
             # We need to remove all the executors of a given topic partition that we have lost, so
@@ -318,7 +330,7 @@ module Karafka
 
         @executors.each do |executor|
           executor.coordinator.increment(:shutdown)
-          job = @jobs_builder.shutdown(executor)
+          job = jobs_builder.shutdown(executor)
           jobs << job
         end
 
@@ -355,7 +367,7 @@ module Karafka
             if coordinator.topic.eofed?
               @executors.find_all_or_create(topic, partition, coordinator).each do |executor|
                 coordinator.increment(:eofed)
-                eofed_jobs << @jobs_builder.eofed(executor)
+                eofed_jobs << jobs_builder.eofed(executor)
               end
             end
 
@@ -372,7 +384,7 @@ module Karafka
             # Start work coordination for this topic partition
             coordinator.increment(:idle)
             executor = @executors.find_or_create(topic, partition, 0, coordinator)
-            idle_jobs << @jobs_builder.idle(executor)
+            idle_jobs << jobs_builder.idle(executor)
 
             next
           end
@@ -383,7 +395,7 @@ module Karafka
           @partitioner.call(topic, messages, coordinator) do |group_id, partition_messages|
             coordinator.increment(:consume)
             executor = @executors.find_or_create(topic, partition, group_id, coordinator)
-            consume_jobs << @jobs_builder.consume(executor, partition_messages)
+            consume_jobs << jobs_builder.consume(executor, partition_messages)
           end
         end
 
@@ -451,7 +463,7 @@ module Karafka
 
             @executors.find_all_or_create(topic_name, partition, coordinator).each do |executor|
               coordinator.increment(:periodic)
-              jobs << @jobs_builder.periodic(executor)
+              jobs << jobs_builder.periodic(executor)
             end
           end
         end

data/lib/karafka/connection/messages_buffer.rb

@@ -45,7 +45,7 @@ module Karafka
         last_polled_at = raw_messages_buffer.last_polled_at
 
         raw_messages_buffer.each do |topic, partition, messages, eof|
-          @size += messages.count
+          @size += messages.size
 
           ktopic = @subscription_group.topics.find(topic)
 

data/lib/karafka/connection/proxy.rb

@@ -108,6 +108,7 @@ module Karafka
       rescue Rdkafka::RdkafkaError => e
         return false if e.code == :assignment_lost
         return false if e.code == :state
+        return false if e.code == :illegal_generation
 
         raise e
       end
@@ -136,6 +137,8 @@ module Karafka
           return false
         when :unknown_member_id
           return false
+        when :illegal_generation
+          return false
         when :no_offset
           return true
         when :coordinator_load_in_progress

data/lib/karafka/constraints.rb

@@ -15,13 +15,13 @@ module Karafka
         # Skip verification if web is not used at all
         return unless require_version('karafka/web')
 
-        # All good if version higher than 0.9.0.rc3 because we expect 0.9.0.rc3 or higher
-        return if version(Karafka::Web::VERSION) >= version('0.9.0.rc3')
+        # All good if version higher than 0.10.0 because we expect 0.10.0 or higher
+        return if version(Karafka::Web::VERSION) >= version('0.10.0')
 
         # If older web-ui used, we cannot allow it
         raise(
           Errors::DependencyConstraintsError,
-          'karafka-web < 0.9.0 is not compatible with this karafka version'
+          'karafka-web < 0.10.0 is not compatible with this karafka version'
         )
       end
 

data/lib/karafka/contracts/config.rb

@@ -35,6 +35,7 @@ module Karafka
       required(:group_id) { |val| val.is_a?(String) && Contracts::TOPIC_REGEXP.match?(val) }
       required(:kafka) { |val| val.is_a?(Hash) && !val.empty? }
       required(:strict_declarative_topics) { |val| [true, false].include?(val) }
+      required(:worker_thread_priority) { |val| (-3..3).to_a.include?(val) }
 
       nested(:swarm) do
         required(:nodes) { |val| val.is_a?(Integer) && val.positive? }
@@ -81,6 +82,7 @@ module Karafka
           required(:manager) { |val| !val.nil? }
           required(:conductor) { |val| !val.nil? }
           required(:reset_backoff) { |val| val.is_a?(Integer) && val >= 1_000 }
+          required(:listener_thread_priority) { |val| (-3..3).to_a.include?(val) }
 
           nested(:proxy) do
             nested(:commit) do
@@ -114,6 +116,7 @@ module Karafka
           required(:jobs_queue_class) { |val| !val.nil? }
           required(:scheduler_class) { |val| !val.nil? }
           required(:coordinator_class) { |val| !val.nil? }
+          required(:errors_tracker_class) { |val| val.nil? || val.is_a?(Class) }
           required(:partitioner_class) { |val| !val.nil? }
           required(:strategy_selector) { |val| !val.nil? }
           required(:expansions_selector) { |val| !val.nil? }

data/lib/karafka/contracts/topic.rb

@@ -70,7 +70,7 @@ module Karafka
         next unless ::Karafka::App.config.strict_topics_namespacing
 
         value = data.fetch(:name)
-        namespacing_chars_count = value.chars.find_all { |c| ['.', '_'].include?(c) }.uniq.count
+        namespacing_chars_count = value.chars.find_all { |c| ['.', '_'].include?(c) }.uniq.size
 
         next if namespacing_chars_count <= 1
 

data/lib/karafka/errors.rb

@@ -22,7 +22,34 @@ module Karafka
     InvalidConfigurationError = Class.new(BaseError)
 
     # Raised when we try to use Karafka CLI commands (except install) without a boot file
-    MissingBootFileError = Class.new(BaseError)
+    MissingBootFileError = Class.new(BaseError) do
+      # @param boot_file_path [Pathname] path where the boot file should be
+      def initialize(boot_file_path)
+        message = <<~MSG
+
+          \e[31mKarafka Boot File Missing:\e[0m #{boot_file_path}
+
+          Cannot find Karafka boot file - this file configures your Karafka application.
+
+          \e[33mQuick fixes:\e[0m
+            \e[32m1.\e[0m Navigate to your Karafka app directory
+            \e[32m2.\e[0m Check if following file exists: \e[36m#{boot_file_path}\e[0m
+            \e[32m3.\e[0m Install Karafka if needed: \e[36mkarafka install\e[0m
+
+          \e[33mCommon causes:\e[0m
+            \e[31m•\e[0m Wrong directory (not in Karafka app root)
+            \e[31m•\e[0m File was accidentally moved or deleted
+            \e[31m•\e[0m New project needing initialization
+
+          For setup help: \e[34mhttps://karafka.io/docs/Getting-Started\e[0m
+        MSG
+
+        super(message)
+        # In case of this error backtrace is irrelevant and we want to print comprehensive error
+        # message without backtrace, this is why nullified.
+        set_backtrace([])
+      end
+    end
 
     # Raised when we've waited enough for shutting down a non-responsive process
     ForcefulShutdownError = Class.new(BaseError)
@@ -35,6 +62,9 @@ module Karafka
     # Raised when given topic is not found while expected
     TopicNotFoundError = Class.new(BaseError)
 
+    # Raised when given consumer group is not found while expected
+    ConsumerGroupNotFoundError = Class.new(BaseError)
+
     # This should never happen. Please open an issue if it does.
     UnsupportedCaseError = Class.new(BaseError)
 
@@ -62,7 +92,17 @@ module Karafka
     ResultNotVisibleError = Class.new(BaseError)
 
     # Raised when there is an attempt to run an unrecognized CLI command
-    UnrecognizedCommandError = Class.new(BaseError)
+    UnrecognizedCommandError = Class.new(BaseError) do
+      # Overwritten not to print backtrace for unknown CLI command
+      def initialize(*args)
+        super
+        set_backtrace([])
+      end
+    end
+
+    # Raised when you were executing a command and it could not finish successfully because of
+    # a setup state or parameters configuration
+    CommandValidationError = Class.new(BaseError)
 
     # Raised when we attempt to perform operation that is only allowed inside of a transaction and
     # there is no transaction around us
@@ -71,6 +111,10 @@ module Karafka
     # Raised in case user would want to perform nested transactions.
     TransactionAlreadyInitializedError = Class.new(BaseError)
 
+    # Raised when user used transactional offset marking but after that tried to use
+    # non-transactional marking, effectively mixing both. This is not allowed.
+    NonTransactionalMarkingAttemptError = Class.new(BaseError)
+
     # Raised in case a listener that was paused is being resumed
     InvalidListenerResumeError = Class.new(BaseError)
 

data/lib/karafka/helpers/async.rb

@@ -37,12 +37,14 @@ module Karafka
 
       # Runs the `#call` method in a new thread
       # @param thread_name [String] name that we want to assign to the thread when we start it
-      def async_call(thread_name)
+      # @param thread_priority [Integer] Ruby thread priority
+      def async_call(thread_name, thread_priority = 0)
         MUTEX.synchronize do
           return if @thread&.alive?
 
           @thread = Thread.new do
             Thread.current.name = thread_name
+            Thread.current.priority = thread_priority
 
             Thread.current.abort_on_exception = true
 

data/lib/karafka/instrumentation/callbacks/rebalance.rb

@@ -12,8 +12,10 @@ module Karafka
 
         # @param subscription_group [Karafka::Routes::SubscriptionGroup] subscription group for
         #   which we want to manage rebalances
-        def initialize(subscription_group)
+        # @param client_id [String] id of the client managing this rebalance
+        def initialize(subscription_group, client_id)
           @subscription_group = subscription_group
+          @client_id = client_id
         end
 
         # Publishes an event that partitions are going to be revoked.
@@ -62,6 +64,7 @@ module Karafka
             subscription_group: @subscription_group,
             consumer_group_id: @subscription_group.consumer_group.id,
             consumer_group: @subscription_group.consumer_group,
+            client_id: @client_id,
             tpl: tpl
           )
         rescue StandardError => e
@@ -71,6 +74,7 @@ module Karafka
             subscription_group_id: @subscription_group.id,
             consumer_group_id: @subscription_group.consumer_group.id,
             type: "callbacks.rebalance.#{name}.error",
+            client_id: @client_id,
             error: e
           )
         end

data/lib/karafka/instrumentation/logger_listener.rb

@@ -24,14 +24,29 @@ module Karafka
         @log_polling = log_polling
       end
 
+      #
+      #
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_connection_listener_before_fetch_loop(event)
+        listener_id = event[:caller].id
+        subscription_group = event[:subscription_group]
+        consumer_group_id = subscription_group.consumer_group.id
+        topics = subscription_group.topics.select(&:active?).map(&:name).join(', ')
+        group_details = "#{consumer_group_id}/#{subscription_group.id}"
+
+        info(
+          "[#{listener_id}] Group #{group_details} subscribing to topics: #{topics}"
+        )
+      end
+
       # Logs each messages fetching attempt
       #
       # @param event [Karafka::Core::Monitoring::Event] event details including payload
       def on_connection_listener_fetch_loop(event)
         return unless log_polling?
 
-        listener = event[:caller]
-        debug "[#{listener.id}] Polling messages..."
+        listener_id = event[:caller].id
+        debug "[#{listener_id}] Polling messages..."
       end
 
       # Logs about messages that we've received from Kafka
@@ -40,11 +55,11 @@ module Karafka
       def on_connection_listener_fetch_loop_received(event)
         return unless log_polling?
 
-        listener = event[:caller]
+        listener_id = event[:caller].id
         time = event[:time].round(2)
         messages_count = event[:messages_buffer].size
 
-        message = "[#{listener.id}] Polled #{messages_count} messages in #{time}ms"
+        message = "[#{listener_id}] Polled #{messages_count} messages in #{time}ms"
 
         # We don't want the "polled 0" in dev as it would spam the log
         # Instead we publish only info when there was anything we could poll and fail over to the
@@ -147,7 +162,8 @@ module Karafka
       #
       # @param event [Karafka::Core::Monitoring::Event] event details including payload
       def on_process_notice_signal(event)
-        info "Received #{event[:signal]} system signal"
+        server_id = Karafka::Server.id
+        info "[#{server_id}] Received #{event[:signal]} system signal"
 
         # We print backtrace only for ttin
         return unless event[:signal] == :SIGTTIN
@@ -168,38 +184,76 @@ module Karafka
 
       # Logs info that we're running Karafka app.
       #
-      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
-      def on_app_running(_event)
-        info "Running in #{RUBY_DESCRIPTION}"
-        info "Running Karafka #{Karafka::VERSION} server"
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_app_running(event)
+        server_id = event[:server_id]
+
+        info "[#{server_id}] Running in #{RUBY_DESCRIPTION}"
+        info "[#{server_id}] Running Karafka #{Karafka::VERSION} server"
 
         return if Karafka.pro?
 
-        info 'See LICENSE and the LGPL-3.0 for licensing details'
+        info "[#{server_id}] See LICENSE and the LGPL-3.0 for licensing details"
       end
 
-      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
-      def on_app_quieting(_event)
-        info 'Switching to quiet mode. New messages will not be processed'
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_app_quieting(event)
+        info "[#{event[:server_id]}] Switching to quiet mode. New messages will not be processed"
       end
 
-      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
-      def on_app_quiet(_event)
-        info 'Reached quiet mode. No messages will be processed anymore'
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_app_quiet(event)
+        info "[#{event[:server_id]}] Reached quiet mode. No messages will be processed anymore"
       end
 
       # Logs info that we're going to stop the Karafka server.
       #
-      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
-      def on_app_stopping(_event)
-        info 'Stopping Karafka server'
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_app_stopping(event)
+        info "[#{event[:server_id]}] Stopping Karafka server"
       end
 
       # Logs info that we stopped the Karafka server.
       #
-      # @param _event [Karafka::Core::Monitoring::Event] event details including payload
-      def on_app_stopped(_event)
-        info 'Stopped Karafka server'
+      # @param event [Karafka::Core::Monitoring::Event] event details including payload
+      def on_app_stopped(event)
+        info "[#{event[:server_id]}] Stopped Karafka server"
+      end
+
+      # Logs info about partitions we have lost
+      #
+      # @param event [Karafka::Core::Monitoring::Event] event details with revoked partitions
+      def on_rebalance_partitions_revoked(event)
+        revoked_partitions = event[:tpl].to_h.transform_values { |part| part.map(&:partition) }
+        group_id = event[:consumer_group_id]
+        client_id = event[:client_id]
+        group_prefix = "[#{client_id}] Group #{group_id} rebalance"
+
+        if revoked_partitions.empty?
+          info "#{group_prefix}: No partitions revoked"
+        else
+          revoked_partitions.each do |topic, partitions|
+            info "#{group_prefix}: Partition(s) #{partitions.join(', ')} of #{topic} revoked"
+          end
+        end
+      end
+
+      # Logs info about partitions that we've gained
+      #
+      # @param event [Karafka::Core::Monitoring::Event] event details with assigned partitions
+      def on_rebalance_partitions_assigned(event)
+        assigned_partitions = event[:tpl].to_h.transform_values { |part| part.map(&:partition) }
+        group_id = event[:consumer_group_id]
+        client_id = event[:client_id]
+        group_prefix = "[#{client_id}] Group #{group_id} rebalance"
+
+        if assigned_partitions.empty?
+          info "#{group_prefix}: No partitions assigned"
+        else
+          assigned_partitions.each do |topic, partitions|
+            info "#{group_prefix}: Partition(s) #{partitions.join(', ')} of #{topic} assigned"
+          end
+        end
       end
 
       # Logs info when we have dispatched a message the the DLQ
@@ -371,9 +425,18 @@ module Karafka
         when 'connection.client.unsubscribe.error'
           error "Client unsubscribe error occurred: #{error}"
           error details
+        when 'parallel_segments.reducer.error'
+          error "Parallel segments reducer error occurred: #{error}"
+          error details
+        when 'parallel_segments.partitioner.error'
+          error "Parallel segments partitioner error occurred: #{error}"
+          error details
+        when 'virtual_partitions.partitioner.error'
+          error "Virtual partitions partitioner error occurred: #{error}"
+          error details
         # This handles any custom errors coming from places like Web-UI, etc
         else
-          error "#{type} error occurred: #{error}"
+          error "#{type} error occurred: #{error.class} - #{error}"
           error details
         end
       end

data/lib/karafka/instrumentation/proctitle_listener.rb

@@ -4,6 +4,10 @@ module Karafka
   module Instrumentation
     # Listener that sets a proc title with a nice descriptive value
     class ProctitleListener
+      include Helpers::ConfigImporter.new(
+        client_id: %i[client_id]
+      )
+
       Status::STATES.each_key do |state|
         class_eval <<~RUBY, __FILE__, __LINE__ + 1
           # Updates proc title to an appropriate state
@@ -19,7 +23,7 @@ module Karafka
       # @param status [String] any status we want to set
       def setproctitle(status)
         ::Process.setproctitle(
-          "karafka #{Karafka::App.config.client_id} (#{status})"
+          "karafka #{client_id} (#{status})"
         )
       end
     end

data/lib/karafka/instrumentation/vendors/datadog/metrics_listener.rb

@@ -131,11 +131,11 @@ module Karafka
             tags = consumer_tags(consumer)
             tags.concat(default_tags)
 
-            count('consumer.messages', messages.count, tags: tags)
+            count('consumer.messages', messages.size, tags: tags)
             count('consumer.batches', 1, tags: tags)
             gauge('consumer.offset', metadata.last_offset, tags: tags)
             histogram('consumer.consumed.time_taken', event[:time], tags: tags)
-            histogram('consumer.batch_size', messages.count, tags: tags)
+            histogram('consumer.batch_size', messages.size, tags: tags)
             histogram('consumer.processing_lag', metadata.processing_lag, tags: tags)
             histogram('consumer.consumption_lag', metadata.consumption_lag, tags: tags)
           end

data/lib/karafka/messages/builders/batch_metadata.rb

@@ -18,7 +18,7 @@ module Karafka
           #   picked up for processing.
           def call(messages, topic, partition, scheduled_at)
             Karafka::Messages::BatchMetadata.new(
-              size: messages.count,
+              size: messages.size,
               first_offset: messages.first&.offset || -1001,
               last_offset: messages.last&.offset || -1001,
               deserializers: topic.deserializers,

data/lib/karafka/pro/cleaner.rb

@@ -28,6 +28,14 @@ module Karafka
         def post_setup(_config)
           true
         end
+
+        # This feature does not need any changes post-fork
+        #
+        # @param _config [Karafka::Core::Configurable::Node]
+        # @param _pre_fork_producer [WaterDrop::Producer]
+        def post_fork(_config, _pre_fork_producer)
+          true
+        end
       end
     end
   end

data/lib/karafka/pro/cli/parallel_segments/base.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Cli
+      class ParallelSegments < Karafka::Cli::Base
+        # Base class for all the parallel segments related operations
+        class Base
+          include Helpers::Colorize
+
+          # @param options [Hash] cli flags options
+          def initialize(options)
+            @options = options
+          end
+
+          private
+
+          # @return [Hash]
+          attr_reader :options
+
+          # Returns consumer groups for parallel segments with which we should be working
+          #
+          # @return [Hash<String, Array<Karafka::Routing::ConsumerGroup>>] hash with all parallel
+          #   consumer groups as values and names of segments origin consumer group as the key.
+          def applicable_groups
+            requested_groups = options[:groups].dup || []
+
+            workable_groups = ::Karafka::App
+                              .routes
+                              .select(&:parallel_segments?)
+                              .group_by(&:segment_origin)
+
+            # Use all if none provided
+            return workable_groups if requested_groups.empty?
+
+            applicable_groups = {}
+
+            requested_groups.each do |requested_group|
+              workable_group = workable_groups[requested_group]
+
+              if workable_group
+                requested_groups.delete(requested_group)
+                applicable_groups[requested_group] = workable_group
+              else
+                raise(
+                  ::Karafka::Errors::ConsumerGroupNotFoundError,
+                  "Consumer group #{requested_group} was not found"
+                )
+              end
+            end
+
+            applicable_groups
+          end
+
+          # Collects the offsets for the segment origin consumer group and the parallel segments
+          # consumers groups. We use segment origin cg offsets as a baseline for the distribution
+          # and use existing (if any) parallel segments cgs offsets for validations.
+          #
+          # @param segment_origin [String] name of the origin consumer group
+          # @param segments [Array<Karafka::Routing::ConsumerGroup>]
+          # @return [Hash] fetched offsets for all the cg topics for all the consumer groups
+          def collect_offsets(segment_origin, segments)
+            topics_names = segments.first.topics.map(&:name)
+            consumer_groups = [segment_origin, segments.map(&:name)].flatten
+
+            consumer_groups_with_topics = consumer_groups
+                                          .map { |name| [name, topics_names] }
+                                          .to_h
+
+            lags_with_offsets = Karafka::Admin.read_lags_with_offsets(
+              consumer_groups_with_topics
+            )
+
+            lags_with_offsets.each do |_cg_name, topics|
+              topics.each do |_topic_name, partitions|
+                partitions.transform_values! { |details| details[:offset] }
+              end
+            end
+
+            lags_with_offsets
+          end
+        end
+      end
+    end
+  end
+end