search-engine-for-typesense 30.1.8.2 → 30.1.8.4

data/lib/search_engine/dependency_planner.rb

@@ -0,0 +1,241 @@
+# frozen_string_literal: true
+
+module SearchEngine
+  # Public dependency-ordering helpers for Typesense collection reference graphs.
+  module DependencyPlanner
+    class << self
+      # Build a normalized reverse dependency graph.
+      # @param source [Symbol] :registry, :typesense, or :auto
+      # @param client [SearchEngine::Client, nil]
+      # @return [Hash{String=>Array<Hash>}] target collection => reference edges
+      def reverse_graph(source: :registry, client: nil)
+        graph = case source.to_sym
+                when :registry
+                  SearchEngine::Cascade.send(:build_from_registry)
+                when :typesense
+                  SearchEngine::Cascade.send(:build_from_typesense, client || SearchEngine.client)
+                when :auto
+                  SearchEngine::Cascade.build_reverse_graph(client: client || SearchEngine.client)
+                else
+                  raise ArgumentError, 'source must be :registry, :typesense, or :auto'
+                end
+
+        normalize_reverse_graph(graph)
+      end
+
+      # Order collection names so referenced collections precede referrers.
+      # @param collections [Array<Symbol, String, Class>] collections or model classes
+      # @param source [Symbol] :registry, :typesense, or :auto
+      # @param client [SearchEngine::Client, nil]
+      # @param reverse_graph [Hash, nil] prebuilt reverse graph
+      # @return [Array<String>]
+      def order_collections(collections, source: :registry, client: nil, reverse_graph: nil)
+        graph = graph_or_build(reverse_graph, source: source, client: client)
+        topo_sort_subset(graph, normalize_collections(collections))
+      end
+
+      # Order events by their collection dependency order while preserving per-collection event order.
+      # @param events [Array<Object, Hash>]
+      # @param collection_method [Symbol, String] event reader or hash key for collection name
+      # @param source [Symbol] :registry, :typesense, or :auto
+      # @param client [SearchEngine::Client, nil]
+      # @param reverse_graph [Hash, nil] prebuilt reverse graph
+      # @return [Array<Object, Hash>]
+      def order_events(events, collection_method: :collection, source: :registry, client: nil, reverse_graph: nil)
+        graph = graph_or_build(reverse_graph, source: source, client: client)
+        grouped = Hash.new { |h, k| h[k] = [] }
+        without_collection = []
+
+        Array(events).each do |event|
+          collection = collection_from_event(event, collection_method)
+          if collection.nil? || collection.empty?
+            without_collection << event
+          else
+            grouped[collection] << event
+          end
+        end
+
+        ordered_collections = topo_sort_subset(graph, grouped.keys)
+        ordered_collections.flat_map { |collection| grouped[collection] } + without_collection
+      end
+
+      # Return collections that directly reference the given collection.
+      # @param collection [Symbol, String, Class]
+      # @param source [Symbol] :registry, :typesense, or :auto
+      # @param client [SearchEngine::Client, nil]
+      # @param reverse_graph [Hash, nil] prebuilt reverse graph
+      # @return [Array<String>]
+      def referencers_for(collection, source: :registry, client: nil, reverse_graph: nil)
+        graph = graph_or_build(reverse_graph, source: source, client: client)
+        name = normalize_collection(collection)
+        Array(graph[name]).filter_map { |edge| edge[:referrer] }.uniq
+      end
+
+      # Return collections directly referenced by the given collection.
+      # @param collection [Symbol, String, Class]
+      # @param source [Symbol] :registry, :typesense, or :auto
+      # @param client [SearchEngine::Client, nil]
+      # @param reverse_graph [Hash, nil] prebuilt reverse graph
+      # @return [Array<String>]
+      def dependencies_for(collection, source: :registry, client: nil, reverse_graph: nil)
+        graph = graph_or_build(reverse_graph, source: source, client: client)
+        name = normalize_collection(collection)
+        deps = []
+        graph.each do |target, edges|
+          deps << target if Array(edges).any? { |edge| edge[:referrer] == name }
+        end
+        deps.uniq
+      end
+
+      # Build the two Bulk stages from collection dependencies.
+      # @param collections [Array<Symbol, String, Class>] collections or model classes
+      # @param source [Symbol] :registry, :typesense, or :auto
+      # @param client [SearchEngine::Client, nil]
+      # @param reverse_graph [Hash, nil] prebuilt reverse graph
+      # @return [Hash{Symbol=>Array<String>}] :stage_1 and :cascade collection names
+      def bulk_stages(collections, source: :auto, client: nil, reverse_graph: nil)
+        inputs = normalize_collections(collections)
+        graph = graph_or_build(reverse_graph, source: source, client: client)
+        input_set = inputs.to_h { |name| [name, true] }
+        internal_referrers = internal_referrers_within_inputs(graph, input_set)
+        cascade_candidates = unique_referencers_of_inputs(graph, inputs)
+
+        {
+          stage_1: inputs.reject { |name| internal_referrers.include?(name) },
+          cascade: topo_sort_subset(graph, cascade_candidates)
+        }
+      end
+
+      private
+
+      def graph_or_build(graph, source:, client:)
+        graph ? normalize_reverse_graph(graph) : reverse_graph(source: source, client: client)
+      end
+
+      def normalize_reverse_graph(graph)
+        normalized = Hash.new { |h, k| h[k] = [] }
+        Hash(graph).each do |target, edges|
+          target_name = target.to_s
+          next if target_name.empty?
+
+          Array(edges).each do |edge|
+            normalized[target_name] << normalize_edge(edge)
+          end
+        end
+        normalized
+      end
+
+      def normalize_edge(edge)
+        {
+          referrer: edge_value(edge, :referrer).to_s,
+          local_key: edge_value(edge, :local_key).to_s,
+          foreign_key: edge_value(edge, :foreign_key).to_s
+        }
+      end
+
+      def edge_value(edge, key)
+        return edge.public_send(key) if edge.respond_to?(key)
+
+        edge[key] || edge[key.to_s]
+      end
+
+      def normalize_collections(collections)
+        Array(collections).flatten.compact.filter_map do |collection|
+          name = normalize_collection(collection)
+          name unless name.empty?
+        end.uniq
+      end
+
+      def normalize_collection(collection)
+        if collection.is_a?(Class)
+          collection.respond_to?(:collection) ? collection.collection.to_s : collection.name.to_s
+        else
+          collection.to_s
+        end
+      end
+
+      def collection_from_event(event, collection_method)
+        if event.respond_to?(collection_method)
+          event.public_send(collection_method).to_s
+        elsif event.respond_to?(:[])
+          (event[collection_method.to_sym] || event[collection_method.to_s]).to_s
+        else
+          ''
+        end
+      end
+
+      def internal_referrers_within_inputs(reverse_graph, input_set)
+        require 'set'
+        refs = Set.new
+        reverse_graph.each do |target, edges|
+          next unless input_set[target]
+
+          Array(edges).each do |edge|
+            referrer = edge[:referrer].to_s
+            refs.add(referrer) if input_set[referrer]
+          end
+        end
+        refs
+      end
+
+      def unique_referencers_of_inputs(reverse_graph, inputs)
+        require 'set'
+        seen = Set.new
+        inputs.each do |name|
+          Array(reverse_graph[name]).each do |edge|
+            referrer = edge[:referrer].to_s
+            seen.add(referrer) unless referrer.empty?
+          end
+        end
+        seen.to_a
+      end
+
+      def topo_sort_subset(reverse_graph, subset)
+        require 'set'
+        nodes = Array(subset).uniq
+        node_set = nodes.to_h { |name| [name, true] }
+        adj = Hash.new { |h, k| h[k] = Set.new }
+        indeg = Hash.new(0)
+
+        nodes.each { |name| indeg[name] = 0 }
+
+        reverse_graph.each do |target, edges|
+          Array(edges).each do |edge|
+            referrer = edge[:referrer].to_s
+            next unless node_set[referrer] && node_set[target]
+            next if adj[target].include?(referrer)
+
+            adj[target] << referrer
+            indeg[referrer] += 1
+          end
+        end
+
+        queue = nodes.select { |name| indeg[name].to_i <= 0 }.sort
+        order = []
+        until queue.empty?
+          name = queue.shift
+          order << name
+          adj[name].each do |dependent|
+            indeg[dependent] -= 1
+            queue << dependent if indeg[dependent] <= 0
+          end
+          queue.sort!
+        end
+
+        remaining = nodes - order
+        instrument_cycle!(remaining) if remaining.any?
+        order + remaining.sort
+      end
+
+      def instrument_cycle!(collections)
+        payload = { collections: collections.sort }
+        SearchEngine::Instrumentation.instrument('search_engine.dependency_planner.cycle', payload) {}
+        SearchEngine.config.logger&.warn(
+          "search_engine dependency planner cycle detected: #{payload[:collections].join(', ')}"
+        )
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end

data/lib/search_engine/indexer/bulk_import.rb

@@ -66,42 +66,25 @@ module SearchEngine
           client = SearchEngine.client
           buffer = +''
           next_index = sequence_generator
-
-          batches = []
-          docs_total = 0
-          success_total = 0
-          failed_total = 0
-          failed_batches_total = 0
-          batches_total = 0
-          source_batches_done = 0
+          import_context = {
+            client: client,
+            collection: into,
+            action: action,
+            retry_policy: retry_policy,
+            buffer: buffer,
+            next_index: next_index
+          }
+          state = initialize_sequential_state
           started_at = monotonic_ms
 
           docs_enum.each do |raw_batch|
-            stats_list = import_batch_with_handling(
-              client: client,
-              collection: into,
+            process_single_batch_sequential(
+              import_context,
               raw_batch: raw_batch,
-              action: action,
-              retry_policy: retry_policy,
-              buffer: buffer,
-              next_index: next_index
-            )
-
-            stats_list.each do |stats|
-              docs_total += stats[:docs_count].to_i
-              success_total += stats[:success_count].to_i
-              failed_total += stats[:failure_count].to_i
-              failed_batches_total += 1 if stats[:failure_count].to_i.positive?
-              batches_total += 1
-              batches << stats
-              validate_soft_batch_size!(batch_size, stats[:docs_count])
-              log_batch(stats, batches_total) if log_batches
-            end
-
-            source_batches_done += 1
-            on_batch&.call(
-              batches_done: source_batches_done, docs_total: docs_total,
-              success_total: success_total, failed_total: failed_total
+              batch_size: batch_size,
+              log_batches: log_batches,
+              on_batch: on_batch,
+              state: state
             )
           end
 
@@ -110,17 +93,56 @@ module SearchEngine
 
           Summary.new(
             collection: klass.respond_to?(:collection) ? klass.collection : klass.name.to_s,
-            status: status_from_counts(success_total, failed_total),
-            batches_total: batches_total,
-            docs_total: docs_total,
-            success_total: success_total,
-            failed_total: failed_total,
-            failed_batches_total: failed_batches_total,
+            status: status_from_counts(state[:success_total], state[:failed_total]),
+            batches_total: state[:batches_total],
+            docs_total: state[:docs_total],
+            success_total: state[:success_total],
+            failed_total: state[:failed_total],
+            failed_batches_total: state[:failed_batches_total],
             duration_ms_total: total_duration_ms,
-            batches: batches
+            source_duration_ms_total: state[:source_duration_ms_total].round(1),
+            map_duration_ms_total: state[:map_duration_ms_total].round(1),
+            jsonl_duration_ms_total: state[:jsonl_duration_ms_total].round(1),
+            import_duration_ms_total: state[:import_duration_ms_total].round(1),
+            batches: state[:batches]
           )
         end
 
+        # @return [Hash] sequential import counters using the same keys as parallel aggregation
+        def initialize_sequential_state
+          {
+            batches: [],
+            docs_total: 0,
+            success_total: 0,
+            failed_total: 0,
+            failed_batches_total: 0,
+            batches_total: 0,
+            source_duration_ms_total: 0.0,
+            map_duration_ms_total: 0.0,
+            jsonl_duration_ms_total: 0.0,
+            import_duration_ms_total: 0.0,
+            source_batches_done: 0
+          }
+        end
+
+        def process_single_batch_sequential(import_context, raw_batch:, batch_size:, log_batches:, on_batch:, state:)
+          stage_metrics = stage_metrics_for(raw_batch)
+          stats_list = import_batch_with_handling(
+            client: import_context.fetch(:client),
+            collection: import_context.fetch(:collection),
+            raw_batch: raw_batch,
+            action: import_context.fetch(:action),
+            retry_policy: import_context.fetch(:retry_policy),
+            buffer: import_context.fetch(:buffer),
+            next_index: import_context.fetch(:next_index)
+          )
+
+          aggregate_stats(stats_list, state, batch_size, log_batches)
+          aggregate_stage_metrics(stage_metrics, stats_list, state)
+          state[:source_batches_done] += 1
+          on_batch&.call(**progress_snapshot(state))
+        end
+
         # Process batches in parallel using a thread pool.
         #
         # Materializes all batches upfront and processes them concurrently using
@@ -202,6 +224,10 @@ module SearchEngine
             failed_total: 0,
             failed_batches_total: 0,
             batches_total: 0,
+            source_duration_ms_total: 0.0,
+            map_duration_ms_total: 0.0,
+            jsonl_duration_ms_total: 0.0,
+            import_duration_ms_total: 0.0,
             source_batches_done: 0,
             idx_counter: -1,
             started_at: monotonic_ms,
@@ -325,6 +351,7 @@ module SearchEngine
           thread_client = SearchEngine.client
           thread_buffer = +''
           thread_idx = shared_state[:mtx].synchronize { shared_state[:idx_counter] += 1 }
+          stage_metrics = stage_metrics_for(raw_batch)
 
           snapshot = begin
             stats_list = import_batch_with_handling(
@@ -339,6 +366,7 @@ module SearchEngine
 
             shared_state[:mtx].synchronize do
               aggregate_stats(stats_list, shared_state, batch_size, log_batches)
+              aggregate_stage_metrics(stage_metrics, stats_list, shared_state)
               shared_state[:source_batches_done] += 1
               progress_snapshot(shared_state)
             end
@@ -355,6 +383,7 @@ module SearchEngine
               err_msg = "  batch_index=#{thread_idx} → error=#{error.class}: #{error.message.to_s[0, 200]}"
               warn(SearchEngine::Logging::Color.apply(err_msg, :red))
               aggregate_stats([failure_stat], shared_state, batch_size, log_batches)
+              aggregate_stage_metrics(stage_metrics, [failure_stat], shared_state)
               shared_state[:source_batches_done] += 1
               progress_snapshot(shared_state)
             end
@@ -420,6 +449,10 @@ module SearchEngine
             failed_total: shared_state[:failed_total],
             failed_batches_total: shared_state[:failed_batches_total],
             duration_ms_total: total_duration_ms,
+            source_duration_ms_total: shared_state[:source_duration_ms_total].round(1),
+            map_duration_ms_total: shared_state[:map_duration_ms_total].round(1),
+            jsonl_duration_ms_total: shared_state[:jsonl_duration_ms_total].round(1),
+            import_duration_ms_total: shared_state[:import_duration_ms_total].round(1),
             batches: shared_state[:batches]
           )
         end
@@ -586,8 +619,10 @@ module SearchEngine
           docs = BatchPlanner.to_array(raw_batch)
           return [] if docs.empty?
 
+          jsonl_started_at = monotonic_ms
           docs_count, bytes_sent = BatchPlanner.encode_jsonl!(docs, buffer)
           jsonl = buffer.dup
+          jsonl_duration_ms = (monotonic_ms - jsonl_started_at).round(1)
           # Use provided batch_index if available (for recursive splits), otherwise compute from next_index
           idx = batch_index || (next_index.is_a?(Proc) ? next_index.call : next_index)
 
@@ -605,6 +640,8 @@ module SearchEngine
             dry_run: false
           )
           stats[:duration_ms] = (monotonic_ms - started_at).round(1)
+          stats[:jsonl_duration_ms] = jsonl_duration_ms
+          stats[:import_duration_ms] = stats[:duration_ms]
           stats[:index] = idx
           [stats]
         rescue Errors::Api => error
@@ -646,11 +683,29 @@ module SearchEngine
             attempts: 1,
             http_status: error&.status.to_i,
             duration_ms: 0.0,
+            jsonl_duration_ms: 0.0,
+            import_duration_ms: 0.0,
             bytes_sent: bytes_sent,
             errors_sample: [safe_error_excerpt(error)]
           }
         end
 
+        def stage_metrics_for(raw_batch)
+          metrics = raw_batch.instance_variable_get(:@__search_engine_stage_metrics__)
+          return metrics if metrics.is_a?(Hash)
+
+          {}
+        rescue StandardError
+          {}
+        end
+
+        def aggregate_stage_metrics(stage_metrics, stats_list, shared_state)
+          shared_state[:source_duration_ms_total] += stage_metrics[:source_duration_ms].to_f
+          shared_state[:map_duration_ms_total] += stage_metrics[:map_duration_ms].to_f
+          shared_state[:jsonl_duration_ms_total] += stats_list.sum { |stats| stats[:jsonl_duration_ms].to_f }
+          shared_state[:import_duration_ms_total] += stats_list.sum { |stats| stats[:import_duration_ms].to_f }
+        end
+
         def safe_error_excerpt(error)
           cls = error&.class&.name
           msg = error&.message.to_s

data/lib/search_engine/indexer.rb

@@ -26,6 +26,10 @@ module SearchEngine
       :failed_total,
       :failed_batches_total,
       :duration_ms_total,
+      :source_duration_ms_total,
+      :map_duration_ms_total,
+      :jsonl_duration_ms_total,
+      :import_duration_ms_total,
       :batches,
       keyword_init: true
     )
@@ -489,6 +493,7 @@ module SearchEngine
       end
 
       def instrument_partition_finish(klass, target_into, pfields, summary, started_at)
+        duration_ms = (monotonic_ms - started_at).round(1)
         SearchEngine::Instrumentation.instrument(
           'search_engine.indexer.partition_finish',
           {
@@ -501,8 +506,12 @@ module SearchEngine
             success_total: summary.success_total,
             failed_total: summary.failed_total,
             status: summary.status,
-            duration_ms: (monotonic_ms - started_at).round(1)
-          }
+            duration_ms: duration_ms,
+            source_duration_ms_total: summary_metric(summary, :source_duration_ms_total),
+            map_duration_ms_total: summary_metric(summary, :map_duration_ms_total),
+            jsonl_duration_ms_total: summary_metric(summary, :jsonl_duration_ms_total),
+            import_duration_ms_total: summary_metric(summary, :import_duration_ms_total)
+          }.compact
         ) {}
       end
 
@@ -510,12 +519,37 @@ module SearchEngine
         Enumerator.new do |y|
           idx = 0
           rows_enum.each do |rows|
+            source_started_at = monotonic_ms
+            rows = BatchPlanner.to_array(rows)
+            source_duration_ms = (monotonic_ms - source_started_at).round(1)
+
+            map_started_at = monotonic_ms
             docs, _report = mapper.map_batch!(rows, batch_index: idx)
+            map_duration_ms = (monotonic_ms - map_started_at).round(1)
+            attach_stage_metrics!(
+              docs,
+              source_duration_ms: source_duration_ms,
+              map_duration_ms: map_duration_ms,
+              source_rows_count: rows.size
+            )
             y << docs
             idx += 1
           end
         end
       end
+
+      def attach_stage_metrics!(docs, metrics)
+        docs.instance_variable_set(:@__search_engine_stage_metrics__, metrics)
+      rescue StandardError
+        nil
+      end
+
+      def summary_metric(summary, key)
+        return unless summary.respond_to?(key)
+
+        value = summary.public_send(key)
+        value&.to_f&.round(1)
+      end
     end
   end
 end

data/lib/search_engine/indexing_run_store/rails_cache.rb

@@ -172,6 +172,11 @@ module SearchEngine
         entry[:success_total] = summary_value(summary, :success_total).to_i
         entry[:failed_total] = summary_value(summary, :failed_total).to_i
         entry[:sample_error] = summary_value(summary, :sample_error)
+        entry[:duration_ms_total] = summary_value(summary, :duration_ms_total)
+        entry[:source_duration_ms_total] = summary_value(summary, :source_duration_ms_total)
+        entry[:map_duration_ms_total] = summary_value(summary, :map_duration_ms_total)
+        entry[:jsonl_duration_ms_total] = summary_value(summary, :jsonl_duration_ms_total)
+        entry[:import_duration_ms_total] = summary_value(summary, :import_duration_ms_total)
       end
 
       def summary_value(summary, key)