search-engine-for-typesense 30.1.6.4 → 30.1.6.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 46b318e4be1f0f8d12e5f3ead35ad4945c472773f8f7117427d1ffd2a0f44732
-  data.tar.gz: 4db7bc9cf0e081ff21b87c8b089627fe02e651020ad8a9df7fc0a3edb0cc7dea
+  metadata.gz: 933b8b58a98c81e8e6b72f3083cfcf2ee1de5ea9f709d4c10aea5d09dba59d99
+  data.tar.gz: 626111c57bc0ee533d21938d7a218c13b80cc23aa27827731994a968f97de672
 SHA512:
-  metadata.gz: cccc05756309485db9c5c49a880e45aa09a3f8f7e6afade62fd2d064eb3ccca286c96791aed830da01090ee10c60b6cf5c602269b55859b3466f76d3d22bd704
-  data.tar.gz: 112cbcd585dfeec20a848c9b153024fcb48526b4f56127a75ef45300693baf6ca666899f607f436e32889d50d2e7f1ad1b29d1daf6af6c5f59b1611db8d9b683
+  metadata.gz: 3e4456f56ddd18865793f0d61890967ba53e50eac9d2b573362800ed0f509aad397bc9233b4f4a0d308e5a9dadfc0a0aac9e9ca946516b638f5eee0a4728fa3b
+  data.tar.gz: 80133e73075e0047968ed9afb3d7756eb5cf47aadeb1c1bbb73d332ee2d44c61e0cc6e6fb5b8932f23ab280f8259319c36d16c288e7d3817a0fdc3844fcde24a

data/lib/search_engine/base/index_maintenance/cleanup.rb

@@ -50,13 +50,14 @@ module SearchEngine
             filters = SearchEngine::StaleRules.compile_filters(self, partition: partition)
             filters.compact!
             filters.reject! { |f| f.to_s.strip.empty? }
+            step = SearchEngine::Logging::StepLine.new('Cleanup')
             if filters.empty?
-              puts(SearchEngine::Logging::Color.dim('Cleanup — skip (no stale configuration)'))
+              step.skip('no stale configuration')
               return 0
             end
 
             merged_filter = SearchEngine::StaleRules.merge_filters(filters)
-            puts("Cleanup — filter=#{merged_filter.inspect}")
+            step.update("filter=#{merged_filter.inspect}")
 
             deleted = SearchEngine::Deletion.delete_by(
               klass: self,
@@ -65,13 +66,14 @@ module SearchEngine
               partition: partition
             )
 
-            puts("Cleanup — #{SearchEngine::Logging::Color.apply("deleted=#{deleted}", :green)}")
+            step.finish("deleted=#{deleted}")
             deleted
           rescue StandardError => error
             err_msg = "Cleanup — error=#{error.class}: #{error.message.to_s[0, 200]}"
             warn(SearchEngine::Logging::Color.apply(err_msg, :red))
             0
           ensure
+            step&.close
             if clear_cache
               begin
                 puts("Cleanup — #{SearchEngine::Logging::Color.bold('cache clear')}")

data/lib/search_engine/base/index_maintenance/lifecycle.rb

@@ -56,12 +56,8 @@ module SearchEngine
 
             diff = SearchEngine::Schema.diff(self, client: client)[:diff] || {}
             missing = __se_schema_missing?(diff)
-            presence = if missing
-                         SearchEngine::Logging::Color.apply('missing', :yellow)
-                       else
-                         SearchEngine::Logging::Color.apply('present', :green)
-                       end
-            puts("Step 1: Presence — #{SearchEngine::Logging::Color.bold('processing')} → #{presence}")
+            step = SearchEngine::Logging::StepLine.new('Presence')
+            missing ? step.finish_warn('missing') : step.finish('present')
 
             applied, indexed_inside_apply = __se_full_apply_if_missing(client, missing)
             drift = __se_full_check_drift(diff, missing, force_rebuild)
@@ -80,78 +76,90 @@ module SearchEngine
           def __se_full_apply_if_missing(client, missing)
             applied = false
             indexed_inside_apply = false
+            step = SearchEngine::Logging::StepLine.new('Schema')
             if missing
-              puts("Step 2: Create+Apply Schema — #{SearchEngine::Logging::Color.bold('processing')}")
+              step.update('creating')
               SearchEngine::Schema.apply!(self, client: client) do |new_physical|
+                step.yield_line!
                 indexed_inside_apply = __se_index_partitions!(into: new_physical)
               end
               applied = true
-              puts("Step 2: Create+Apply Schema — #{SearchEngine::Logging::Color.apply('done', :green)}")
+              step.finish('created')
             else
-              puts(SearchEngine::Logging::Color.dim('Step 2: Create+Apply Schema — skip (collection present)'))
+              step.skip('collection present')
             end
             [applied, indexed_inside_apply]
+          ensure
+            step&.close
           end
 
           def __se_full_check_drift(diff, missing, force_rebuild)
+            step = SearchEngine::Logging::StepLine.new('Schema Status')
             unless missing
-              puts("Step 3: Check Schema Status — #{SearchEngine::Logging::Color.bold('processing')}")
+              step.update('checking')
               drift = __se_schema_drift?(diff)
               if force_rebuild && !drift
-                puts("Step 3: Check Schema Status — #{SearchEngine::Logging::Color.apply('force_rebuild', :yellow)}")
+                step.finish_warn('force_rebuild')
                 return true
               end
-              schema_status = if drift
-                                SearchEngine::Logging::Color.apply('drift', :yellow)
-                              else
-                                SearchEngine::Logging::Color.apply('in_sync', :green)
-                              end
-              puts("Step 3: Check Schema Status — #{schema_status}")
+              drift ? step.finish_warn('drift') : step.finish('in_sync')
               return drift
             end
-            puts(SearchEngine::Logging::Color.dim('Step 3: Check Schema Status — skip (just created)'))
+            step.skip('just created')
             false
+          ensure
+            step&.close
           end
 
           def __se_full_apply_if_drift(client, drift, applied, indexed_inside_apply, force_rebuild)
+            step = SearchEngine::Logging::StepLine.new('Schema Apply')
             if drift
-              puts("Step 4: Apply New Schema — #{SearchEngine::Logging::Color.bold('processing')}")
+              step.update('applying')
               SearchEngine::Schema.apply!(self, client: client, force_rebuild: force_rebuild) do |new_physical|
+                step.yield_line!
                 indexed_inside_apply = __se_index_partitions!(into: new_physical)
               end
               applied = true
-              puts("Step 4: Apply New Schema — #{SearchEngine::Logging::Color.apply('done', :green)}")
+              step.finish('applied')
             else
-              puts(SearchEngine::Logging::Color.dim('Step 4: Apply New Schema — skip'))
+              step.skip
             end
             [applied, indexed_inside_apply]
+          ensure
+            step&.close
           end
 
           def __se_full_indexation(applied, indexed_inside_apply)
             result = nil
+            step = SearchEngine::Logging::StepLine.new('Indexing')
             if applied && indexed_inside_apply
-              puts(SearchEngine::Logging::Color.dim('Step 5: Indexing — skip (performed during schema apply)'))
+              step.skip('performed during schema apply')
               result = indexed_inside_apply if indexed_inside_apply.is_a?(Hash)
             else
-              puts("Step 5: Indexing — #{SearchEngine::Logging::Color.bold('processing')}")
+              step.update('indexing')
+              step.yield_line!
               result = __se_index_partitions!(into: nil)
-              puts("Step 5: Indexing — #{SearchEngine::Logging::Color.apply('done', :green)}")
+              step.finish('done')
             end
 
             cascade_ok = result.is_a?(Hash) ? result[:status] == :ok : false
             __se_cascade_after_indexation!(context: :full) if cascade_ok
             result
+          ensure
+            step&.close
           end
 
           def __se_full_retention(applied, logical, client)
+            step = SearchEngine::Logging::StepLine.new('Retention')
             if applied
-              puts(SearchEngine::Logging::Color.dim('Step 6: Retention Cleanup — skip (handled by schema apply)'))
+              step.skip('handled by schema apply')
             else
-              puts("Step 6: Retention Cleanup — #{SearchEngine::Logging::Color.bold('processing')}")
+              step.update('cleaning')
               dropped = __se_retention_cleanup!(logical: logical, client: client)
-              dropped_str = SearchEngine::Logging::Color.apply("dropped=#{dropped.inspect}", :green)
-              puts("Step 6: Retention Cleanup — #{dropped_str}")
+              step.finish("dropped=#{dropped.inspect}")
             end
+          ensure
+            step&.close
           end
 
           def __se_index_partial(partition:, client:, pre: nil)
@@ -160,47 +168,42 @@ module SearchEngine
             diff = diff_res[:diff] || {}
 
             missing = __se_schema_missing?(diff)
-            presence = if missing
-                         SearchEngine::Logging::Color.apply('missing', :yellow)
-                       else
-                         SearchEngine::Logging::Color.apply('present', :green)
-                       end
-            puts("Step 1: Presence — #{SearchEngine::Logging::Color.bold('processing')} → #{presence}")
+            step = SearchEngine::Logging::StepLine.new('Presence')
             if missing
-              msg = SearchEngine::Logging::Color.apply(
-                'Step 1: Partial — collection is not present. Quit early.', :yellow
-              )
-              puts(msg)
+              step.finish_warn('missing — collection not present, exit early')
               return { status: :failed, docs_total: 0, success_total: 0, failed_total: 0,
                        sample_error: 'Collection not present' }
             end
+            step.finish('present')
 
-            puts("Step 2: Check Schema Status — #{SearchEngine::Logging::Color.bold('processing')}")
+            step = SearchEngine::Logging::StepLine.new('Schema Status')
+            step.update('checking')
             drift = __se_schema_drift?(diff)
             if drift
-              msg = SearchEngine::Logging::Color.apply(
-                'Step 2: Partial — schema is not up-to-date. Exit early (run full indexing).', :yellow
-              )
-              puts(msg)
+              step.finish_warn('drift — exit early (run full indexing)')
               return { status: :failed, docs_total: 0, success_total: 0, failed_total: 0,
                        sample_error: 'Schema drift detected' }
             end
-            puts("Step 2: Check Schema Status — #{SearchEngine::Logging::Color.apply('in_sync', :green)}")
+            step.finish('in_sync')
 
             __se_preflight_dependencies!(mode: pre, client: client) if pre
 
-            puts("Step 3: Partial Indexing — #{SearchEngine::Logging::Color.bold('processing')}")
+            step = SearchEngine::Logging::StepLine.new('Partial Indexing')
+            step.update('indexing')
+            step.yield_line!
             summaries = []
             partitions.each do |p|
               summary = SearchEngine::Indexer.rebuild_partition!(self, partition: p, into: nil)
               summaries << summary
               puts(SearchEngine::Logging::PartitionProgress.line(p, summary))
             end
-            puts("Step 3: Partial Indexing — #{SearchEngine::Logging::Color.apply('done', :green)}")
+            step.finish('done')
 
             result = __se_build_index_result(summaries)
             __se_cascade_after_indexation!(context: :full) if result[:status] == :ok
             result
+          ensure
+            step&.close
           end
 
           # rubocop:disable Metrics/PerceivedComplexity, Metrics/AbcSize

data/lib/search_engine/base/index_maintenance/schema.rb

@@ -27,28 +27,24 @@ module SearchEngine
 
           def update_collection!
             client = SearchEngine.client
-
-            status_word = SearchEngine::Logging::Color.bold('analyzing diff for in-place update...')
-            puts "Update Collection — #{status_word}"
+            step = SearchEngine::Logging::StepLine.new('Update Collection')
+            step.update('analyzing diff')
             updated = SearchEngine::Schema.update!(self, client: client)
 
             if updated
-              status_word = SearchEngine::Logging::Color.apply('schema updated in-place (PATCH)', :green)
-              puts "Update Collection — #{status_word}"
+              step.finish('updated in-place (PATCH)')
             else
-              msg = SearchEngine::Logging::Color.dim(
-                'Update Collection — in-place update not possible (no changes or incompatible)'
-              )
-              puts(msg)
+              step.skip('no changes or incompatible')
             end
             updated
+          ensure
+            step&.close
           end
 
           def drop_collection!
             client = SearchEngine.client
             logical = respond_to?(:collection) ? collection.to_s : name.to_s
 
-            # Resolve alias with a safer timeout for control-plane operations
             alias_target = client.resolve_alias(logical, timeout_ms: 10_000)
             physical = if alias_target && !alias_target.to_s.strip.empty?
                          alias_target.to_s
@@ -57,21 +53,21 @@ module SearchEngine
                          live ? logical : nil
                        end
 
+            step = SearchEngine::Logging::StepLine.new('Drop Collection')
             if physical.nil?
-              puts(SearchEngine::Logging::Color.dim('Drop Collection — skip (not present)'))
+              step.skip('not present')
               return
             end
 
             puts
-            header = SearchEngine::Logging::Color.header(%(>>>>>> Dropping Collection "#{logical}"))
-            puts(header)
-            status_word = SearchEngine::Logging::Color.bold('processing')
-            puts("Drop Collection — #{status_word} (logical=#{logical} physical=#{physical})")
-            # Use an extended timeout to accommodate large collection drops
+            puts(SearchEngine::Logging::Color.header(%(>>>>>> Dropping Collection "#{logical}")))
+            step.update("dropping (logical=#{logical} physical=#{physical})")
             client.delete_collection(physical, timeout_ms: 60_000)
-            puts("Drop Collection — #{SearchEngine::Logging::Color.apply('done', :green)}")
+            step.finish('done')
             puts(SearchEngine::Logging::Color.header(%(>>>>>> Dropped Collection "#{logical}")))
             nil
+          ensure
+            step&.close
           end
 
           def recreate_collection!
@@ -86,21 +82,21 @@ module SearchEngine
                          live ? logical : nil
                        end
 
+            step = SearchEngine::Logging::StepLine.new('Recreate Collection')
             if physical
-              status_word = SearchEngine::Logging::Color.apply('dropping existing', :yellow)
-              puts("Recreate Collection — #{status_word} (logical=#{logical} physical=#{physical})")
+              step.update("dropping existing (logical=#{logical} physical=#{physical})")
               client.delete_collection(physical)
             else
-              msg = SearchEngine::Logging::Color.dim('Recreate Collection — no existing collection (skip drop)')
-              puts(msg)
+              step.update("creating (logical=#{logical})")
             end
 
             schema = SearchEngine::Schema.compile(self)
-            status_word = SearchEngine::Logging::Color.bold('creating collection with schema')
-            puts("Recreate Collection — #{status_word} (logical=#{logical})")
+            step.update("creating with schema (logical=#{logical})")
             client.create_collection(schema)
-            puts("Recreate Collection — #{SearchEngine::Logging::Color.apply('done', :green)}")
+            step.finish('done')
             nil
+          ensure
+            step&.close
           end
 
           def __se_retention_cleanup!(_logical:, _client:)

data/lib/search_engine/base/index_maintenance.rb

@@ -6,6 +6,8 @@ require 'search_engine/base/index_maintenance/lifecycle'
 require 'search_engine/base/index_maintenance/schema'
 require 'search_engine/logging/color'
 require 'search_engine/logging/batch_line'
+require 'search_engine/logging/step_line'
+require 'search_engine/logging/live_renderer'
 
 module SearchEngine
   class Base
@@ -275,6 +277,9 @@ module SearchEngine
             summary = SearchEngine::Indexer.rebuild_partition!(self, partition: nil, into: into)
             __se_build_index_result([summary])
           end
+        rescue StandardError => error
+          { status: :failed, docs_total: 0, success_total: 0, failed_total: 0,
+            sample_error: "#{error.class}: #{error.message.to_s[0, 200]}" }
         end
 
         # Aggregate an array of Indexer::Summary structs into a single result hash.
@@ -308,59 +313,120 @@ module SearchEngine
       end
 
       class_methods do
-        # Sequential processing of partition list
+        # Sequential processing of partition list with live progress rendering.
         def __se_index_partitions_seq!(parts, into)
+          estimate = __se_per_partition_estimate(parts.size)
+          renderer = SearchEngine::Logging::LiveRenderer.new(
+            labels: parts.map(&:inspect), partitions: parts, per_partition_estimate: estimate
+          )
+          renderer.start
+
           summaries = []
-          parts.each do |part|
-            summary = SearchEngine::Indexer.rebuild_partition!(self, partition: part, into: into)
-            summaries << summary
-            puts(SearchEngine::Logging::PartitionProgress.line(part, summary))
-            __se_log_batches_from_summary(summary.batches) if summary.batches_total.to_i > 1
+          parts.each_with_index do |part, idx|
+            slot = renderer[idx]
+            slot.start
+            begin
+              on_batch = ->(info) { slot.progress(**info) }
+              summary = SearchEngine::Indexer.rebuild_partition!(
+                self, partition: part, into: into, on_batch: on_batch
+              )
+              slot.finish(summary)
+              summaries << summary
+            rescue StandardError => error
+              slot.finish_error(error)
+              raise
+            end
+          end
+
+          begin
+            renderer.stop
+          rescue StandardError
+            nil
           end
           __se_build_index_result(summaries)
+        ensure
+          renderer&.stop
         end
       end
 
       class_methods do
-        # Parallel processing via bounded thread pool
+        # Parallel processing via bounded thread pool with live progress rendering.
         def __se_index_partitions_parallel!(parts, into, max_p)
           require 'concurrent-ruby'
+
+          estimate = __se_per_partition_estimate(parts.size)
+          renderer = SearchEngine::Logging::LiveRenderer.new(
+            labels: parts.map(&:inspect), partitions: parts, per_partition_estimate: estimate
+          )
+          renderer.start
+
           pool = Concurrent::FixedThreadPool.new(max_p)
+          cancelled = Concurrent::AtomicBoolean.new(false)
           ctx = SearchEngine::Instrumentation.context
-          mtx = Mutex.new
           summaries = []
           partition_errors = []
-          begin
-            parts.each do |part|
+          mtx = Mutex.new
+
+          on_interrupt = lambda do
+            cancelled.make_true
+            renderer.stop
+            warn("\n  Interrupted \u2014 stopping parallel partition workers\u2026")
+          end
+
+          SearchEngine::InterruptiblePool.run(pool, on_interrupt: on_interrupt) do
+            parts.each_with_index do |part, idx|
+              break if cancelled.true?
+
               pool.post do
+                next if cancelled.true?
+
+                slot = renderer[idx]
+                slot.start
                 SearchEngine::Instrumentation.with_context(ctx) do
-                  summary = SearchEngine::Indexer.rebuild_partition!(self, partition: part, into: into)
-                  mtx.synchronize do
-                    summaries << summary
-                    puts(SearchEngine::Logging::PartitionProgress.line(part, summary))
-                    __se_log_batches_from_summary(summary.batches) if summary.batches_total.to_i > 1
-                  end
+                  on_batch = ->(info) { slot.progress(**info) }
+                  summary = SearchEngine::Indexer.rebuild_partition!(
+                    self, partition: part, into: into, on_batch: on_batch
+                  )
+                  slot.finish(summary)
+                  mtx.synchronize { summaries << summary }
                 end
               rescue StandardError => error
+                renderer[idx].finish_error(error)
                 mtx.synchronize do
-                  err_msg = "  partition=#{part.inspect} → error=#{error.class}: #{error.message.to_s[0, 200]}"
-                  warn(SearchEngine::Logging::Color.apply(err_msg, :red))
-                  partition_errors << "#{error.class}: #{error.message.to_s[0, 200]}"
+                  cancelled.make_true
+                  partition_errors << error unless partition_errors.any?
                 end
               end
             end
-          ensure
-            pool.shutdown
-            pool.wait_for_termination(3600) || pool.kill
-            pool.wait_for_termination(60)
           end
-          result = __se_build_index_result(summaries)
-          if partition_errors.any?
-            result[:status] = :failed
-            result[:sample_error] ||= partition_errors.first
+
+          begin
+            renderer.stop
+          rescue StandardError
+            nil
           end
-          result
+          raise partition_errors.first if partition_errors.any?
+
+          __se_build_index_result(summaries)
+        ensure
+          renderer&.stop
+        end
+      end
+
+      class_methods do
+        # Heuristic per-partition batch estimate for progress bars.
+        # @param partition_count [Integer]
+        # @return [Integer, nil]
+        def __se_per_partition_estimate(partition_count)
+          total = SearchEngine::Indexer::BulkImport.estimate_total_batches(self)
+          return nil unless total
+
+          (total.to_f / partition_count).ceil
+        rescue StandardError
+          nil
         end
+
+        private :__se_per_partition_estimate
       end
 
       class_methods do

data/lib/search_engine/cascade.rb

@@ -145,7 +145,7 @@ into: nil
       # when no partitions are configured.
       # @param ref_klass [Class]
       # @return [void]
-      # rubocop:disable Metrics/PerceivedComplexity, Metrics/AbcSize
+      # rubocop:disable Metrics/AbcSize
       def __se_full_reindex_for_referrer(ref_klass, client:, alias_cache:)
         logical = ref_klass.respond_to?(:collection) ? ref_klass.collection.to_s : ref_klass.name.to_s
         physical = resolve_physical_collection_name(logical, client: client, cache: alias_cache)
@@ -197,13 +197,9 @@ into: nil
             pool = Concurrent::FixedThreadPool.new(mp)
             ctx = SearchEngine::Instrumentation.context
             mtx = Mutex.new
-            begin
+            on_interrupt = -> { warn("\n  Interrupted — stopping parallel cascade workers…") }
+            SearchEngine::InterruptiblePool.run(pool, on_interrupt: on_interrupt) do
               post_partitions_to_pool!(pool, ctx, parts, ref_klass, mtx)
-            ensure
-              pool.shutdown
-              # Wait up to 1 hour, then force-kill and wait a bit more to ensure cleanup
-              pool.wait_for_termination(3600) || pool.kill
-              pool.wait_for_termination(60)
             end
             executed = true
           else
@@ -218,7 +214,7 @@ into: nil
         end
         executed
       end
-      # rubocop:enable Metrics/PerceivedComplexity, Metrics/AbcSize
+      # rubocop:enable Metrics/AbcSize
 
       # Resolve logical alias to physical name with optional per-run memoization.
       # @param logical [String]

data/lib/search_engine/config.rb

@@ -103,6 +103,8 @@ module SearchEngine
       attr_accessor :dispatch
       # @return [String] queue name for ActiveJob enqueues
       attr_accessor :queue_name
+      # @return [Boolean] whether to run model.count for progress bar estimates (default true)
+      attr_accessor :estimate_progress
 
       def initialize
         @batch_size = 2000
@@ -111,6 +113,7 @@ module SearchEngine
         @gzip = false
         @dispatch = active_job_available? ? :active_job : :inline
         @queue_name = 'search_index'
+        @estimate_progress = true
       end
 
       private
@@ -720,7 +723,8 @@ module SearchEngine
         retries: indexer.retries,
         gzip: indexer.gzip ? true : false,
         dispatch: indexer.dispatch,
-        queue_name: indexer.queue_name
+        queue_name: indexer.queue_name,
+        estimate_progress: indexer.estimate_progress
       }
     end