search-engine-for-typesense 30.1.8.2 → 30.1.8.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e6ab09e7d02193a7f30842796381569a79cc89beb95cb274097877d957494e92
-  data.tar.gz: 9ec75fa9b98df1de5c7abbf141f45e56d16e0eac41f2802087515ee31c4bd1fc
+  metadata.gz: 33213d7f97f8014582dc5495606a158d61f8cd72c71d7467c5c98b8b4d5bced3
+  data.tar.gz: 766b1c9e8dfe3e20ad57c32ddfc5a0d3ca2c5fb6dba763f4585f5c27fd461d8f
 SHA512:
-  metadata.gz: f916255418a47e8c8c8aa55dffaa701967ee092f5e2c8e2c6a3bc9f7f0106ad318053681fe2899669af2df0850b1a6b223a418eb301683e66599c5d3eca8c0a1
-  data.tar.gz: f22af0553d99b19fc9ab1f0dda24a8d10be80421844c4282038971e7273babec79d19a995d2465f90cc8a92cc0d470d79b38eadbfd7654599335f18e7fdde0e5
+  metadata.gz: 60603142ea1d344165fe4bf395eb7c395775bd76697247aa8fc79d3a153adb8ffe95f5a2e7a0df0fcd61f39ebe7f8fbcb17077b06752f2aad1a97d91b2bade03
+  data.tar.gz: 505aa6f7cec4aeb51ce7be0b3985ea479679c578ee401e24415a4f32d77ccfb965c95fb95f649daf293d32a41523458bb44d99eec32387f054017a67e507ffd3

data/lib/search_engine/indexer/bulk_import.rb

@@ -73,10 +73,15 @@ module SearchEngine
           failed_total = 0
           failed_batches_total = 0
           batches_total = 0
+          source_duration_ms_total = 0.0
+          map_duration_ms_total = 0.0
+          jsonl_duration_ms_total = 0.0
+          import_duration_ms_total = 0.0
           source_batches_done = 0
           started_at = monotonic_ms
 
           docs_enum.each do |raw_batch|
+            stage_metrics = stage_metrics_for(raw_batch)
             stats_list = import_batch_with_handling(
               client: client,
               collection: into,
@@ -97,6 +102,10 @@ module SearchEngine
               validate_soft_batch_size!(batch_size, stats[:docs_count])
               log_batch(stats, batches_total) if log_batches
             end
+            source_duration_ms_total += stage_metrics[:source_duration_ms].to_f
+            map_duration_ms_total += stage_metrics[:map_duration_ms].to_f
+            jsonl_duration_ms_total += stats_list.sum { |stats| stats[:jsonl_duration_ms].to_f }
+            import_duration_ms_total += stats_list.sum { |stats| stats[:import_duration_ms].to_f }
 
             source_batches_done += 1
             on_batch&.call(
@@ -117,6 +126,10 @@ module SearchEngine
             failed_total: failed_total,
             failed_batches_total: failed_batches_total,
             duration_ms_total: total_duration_ms,
+            source_duration_ms_total: source_duration_ms_total.round(1),
+            map_duration_ms_total: map_duration_ms_total.round(1),
+            jsonl_duration_ms_total: jsonl_duration_ms_total.round(1),
+            import_duration_ms_total: import_duration_ms_total.round(1),
             batches: batches
           )
         end
@@ -202,6 +215,10 @@ module SearchEngine
             failed_total: 0,
             failed_batches_total: 0,
             batches_total: 0,
+            source_duration_ms_total: 0.0,
+            map_duration_ms_total: 0.0,
+            jsonl_duration_ms_total: 0.0,
+            import_duration_ms_total: 0.0,
             source_batches_done: 0,
             idx_counter: -1,
             started_at: monotonic_ms,
@@ -325,6 +342,7 @@ module SearchEngine
           thread_client = SearchEngine.client
           thread_buffer = +''
           thread_idx = shared_state[:mtx].synchronize { shared_state[:idx_counter] += 1 }
+          stage_metrics = stage_metrics_for(raw_batch)
 
           snapshot = begin
             stats_list = import_batch_with_handling(
@@ -339,6 +357,7 @@ module SearchEngine
 
             shared_state[:mtx].synchronize do
               aggregate_stats(stats_list, shared_state, batch_size, log_batches)
+              aggregate_stage_metrics(stage_metrics, stats_list, shared_state)
               shared_state[:source_batches_done] += 1
               progress_snapshot(shared_state)
             end
@@ -355,6 +374,7 @@ module SearchEngine
               err_msg = "  batch_index=#{thread_idx} → error=#{error.class}: #{error.message.to_s[0, 200]}"
               warn(SearchEngine::Logging::Color.apply(err_msg, :red))
               aggregate_stats([failure_stat], shared_state, batch_size, log_batches)
+              aggregate_stage_metrics(stage_metrics, [failure_stat], shared_state)
               shared_state[:source_batches_done] += 1
               progress_snapshot(shared_state)
             end
@@ -420,6 +440,10 @@ module SearchEngine
             failed_total: shared_state[:failed_total],
             failed_batches_total: shared_state[:failed_batches_total],
             duration_ms_total: total_duration_ms,
+            source_duration_ms_total: shared_state[:source_duration_ms_total].round(1),
+            map_duration_ms_total: shared_state[:map_duration_ms_total].round(1),
+            jsonl_duration_ms_total: shared_state[:jsonl_duration_ms_total].round(1),
+            import_duration_ms_total: shared_state[:import_duration_ms_total].round(1),
             batches: shared_state[:batches]
           )
         end
@@ -586,8 +610,10 @@ module SearchEngine
           docs = BatchPlanner.to_array(raw_batch)
           return [] if docs.empty?
 
+          jsonl_started_at = monotonic_ms
           docs_count, bytes_sent = BatchPlanner.encode_jsonl!(docs, buffer)
           jsonl = buffer.dup
+          jsonl_duration_ms = (monotonic_ms - jsonl_started_at).round(1)
           # Use provided batch_index if available (for recursive splits), otherwise compute from next_index
           idx = batch_index || (next_index.is_a?(Proc) ? next_index.call : next_index)
 
@@ -605,6 +631,8 @@ module SearchEngine
             dry_run: false
           )
           stats[:duration_ms] = (monotonic_ms - started_at).round(1)
+          stats[:jsonl_duration_ms] = jsonl_duration_ms
+          stats[:import_duration_ms] = stats[:duration_ms]
           stats[:index] = idx
           [stats]
         rescue Errors::Api => error
@@ -646,11 +674,29 @@ module SearchEngine
             attempts: 1,
             http_status: error&.status.to_i,
             duration_ms: 0.0,
+            jsonl_duration_ms: 0.0,
+            import_duration_ms: 0.0,
             bytes_sent: bytes_sent,
             errors_sample: [safe_error_excerpt(error)]
           }
         end
 
+        def stage_metrics_for(raw_batch)
+          metrics = raw_batch.instance_variable_get(:@__search_engine_stage_metrics__)
+          return metrics if metrics.is_a?(Hash)
+
+          {}
+        rescue StandardError
+          {}
+        end
+
+        def aggregate_stage_metrics(stage_metrics, stats_list, shared_state)
+          shared_state[:source_duration_ms_total] += stage_metrics[:source_duration_ms].to_f
+          shared_state[:map_duration_ms_total] += stage_metrics[:map_duration_ms].to_f
+          shared_state[:jsonl_duration_ms_total] += stats_list.sum { |stats| stats[:jsonl_duration_ms].to_f }
+          shared_state[:import_duration_ms_total] += stats_list.sum { |stats| stats[:import_duration_ms].to_f }
+        end
+
         def safe_error_excerpt(error)
           cls = error&.class&.name
           msg = error&.message.to_s

data/lib/search_engine/indexer.rb

@@ -26,6 +26,10 @@ module SearchEngine
       :failed_total,
       :failed_batches_total,
       :duration_ms_total,
+      :source_duration_ms_total,
+      :map_duration_ms_total,
+      :jsonl_duration_ms_total,
+      :import_duration_ms_total,
       :batches,
       keyword_init: true
     )
@@ -489,6 +493,7 @@ module SearchEngine
       end
 
       def instrument_partition_finish(klass, target_into, pfields, summary, started_at)
+        duration_ms = (monotonic_ms - started_at).round(1)
         SearchEngine::Instrumentation.instrument(
           'search_engine.indexer.partition_finish',
           {
@@ -501,8 +506,12 @@ module SearchEngine
             success_total: summary.success_total,
             failed_total: summary.failed_total,
             status: summary.status,
-            duration_ms: (monotonic_ms - started_at).round(1)
-          }
+            duration_ms: duration_ms,
+            source_duration_ms_total: summary_metric(summary, :source_duration_ms_total),
+            map_duration_ms_total: summary_metric(summary, :map_duration_ms_total),
+            jsonl_duration_ms_total: summary_metric(summary, :jsonl_duration_ms_total),
+            import_duration_ms_total: summary_metric(summary, :import_duration_ms_total)
+          }.compact
         ) {}
       end
 
@@ -510,12 +519,37 @@ module SearchEngine
         Enumerator.new do |y|
           idx = 0
           rows_enum.each do |rows|
+            source_started_at = monotonic_ms
+            rows = BatchPlanner.to_array(rows)
+            source_duration_ms = (monotonic_ms - source_started_at).round(1)
+
+            map_started_at = monotonic_ms
             docs, _report = mapper.map_batch!(rows, batch_index: idx)
+            map_duration_ms = (monotonic_ms - map_started_at).round(1)
+            attach_stage_metrics!(
+              docs,
+              source_duration_ms: source_duration_ms,
+              map_duration_ms: map_duration_ms,
+              source_rows_count: rows.size
+            )
             y << docs
             idx += 1
           end
         end
       end
+
+      def attach_stage_metrics!(docs, metrics)
+        docs.instance_variable_set(:@__search_engine_stage_metrics__, metrics)
+      rescue StandardError
+        nil
+      end
+
+      def summary_metric(summary, key)
+        return unless summary.respond_to?(key)
+
+        value = summary.public_send(key)
+        value.nil? ? nil : value.to_f.round(1)
+      end
     end
   end
 end

data/lib/search_engine/indexing_run_store/rails_cache.rb

@@ -172,6 +172,11 @@ module SearchEngine
         entry[:success_total] = summary_value(summary, :success_total).to_i
         entry[:failed_total] = summary_value(summary, :failed_total).to_i
         entry[:sample_error] = summary_value(summary, :sample_error)
+        entry[:duration_ms_total] = summary_value(summary, :duration_ms_total)
+        entry[:source_duration_ms_total] = summary_value(summary, :source_duration_ms_total)
+        entry[:map_duration_ms_total] = summary_value(summary, :map_duration_ms_total)
+        entry[:jsonl_duration_ms_total] = summary_value(summary, :jsonl_duration_ms_total)
+        entry[:import_duration_ms_total] = summary_value(summary, :import_duration_ms_total)
       end
 
       def summary_value(summary, key)

data/lib/search_engine/version.rb

@@ -3,5 +3,5 @@
 module SearchEngine
   # Current gem version.
   # @return [String]
-  VERSION = '30.1.8.2'
+  VERSION = '30.1.8.3'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: search-engine-for-typesense
 version: !ruby/object:Gem::Version
-  version: 30.1.8.2
+  version: 30.1.8.3
 platform: ruby
 authors:
 - Nikita Shkoda