search-engine-for-typesense 1.0.0

data/lib/search_engine/base/index_maintenance/lifecycle.rb

@@ -0,0 +1,251 @@
+# frozen_string_literal: true
+
+module SearchEngine
+  class Base
+    module IndexMaintenance
+      # Lifecycle orchestration for full/partial indexing flows.
+      module Lifecycle
+        extend ActiveSupport::Concern
+
+        class_methods do
+          # Run indexing workflow for this collection.
+          # @param partition [Object, Array<Object>, nil]
+          # @param client [SearchEngine::Client, nil]
+          # @param pre [Symbol, nil] :ensure (ensure presence) or :index (ensure + fix drift)
+          # @param force_rebuild [Boolean] when true, force schema rebuild (blue/green)
+          # @return [void]
+          def index_collection(partition: nil, client: nil, pre: nil, force_rebuild: false)
+            logical = respond_to?(:collection) ? collection.to_s : name.to_s
+            puts
+            puts(%(>>>>>> Indexing Collection "#{logical}"))
+            client_obj = client || SearchEngine.client
+
+            if partition.nil?
+              __se_index_full(client: client_obj, pre: pre, force_rebuild: force_rebuild)
+            else
+              __se_index_partial(partition: partition, client: client_obj, pre: pre)
+            end
+            nil
+          end
+
+          def reindex_collection!(pre: nil)
+            drop_collection!
+            index_collection(pre: pre)
+          end
+
+          def rebuild_partition!(partition:, into: nil, pre: nil)
+            if pre
+              client_obj = SearchEngine.client
+              __se_preflight_dependencies!(mode: pre, client: client_obj)
+            end
+            parts = if partition.nil? || (partition.respond_to?(:empty?) && partition.empty?)
+                      [nil]
+                    else
+                      Array(partition)
+                    end
+
+            return SearchEngine::Indexer.rebuild_partition!(self, partition: parts.first, into: into) if parts.size == 1
+
+            parts.map { |p| SearchEngine::Indexer.rebuild_partition!(self, partition: p, into: into) }
+          end
+
+          def __se_index_full(client:, pre: nil, force_rebuild: false)
+            logical = respond_to?(:collection) ? collection.to_s : name.to_s
+            __se_preflight_dependencies!(mode: pre, client: client) if pre
+
+            diff = SearchEngine::Schema.diff(self, client: client)[:diff] || {}
+            missing = __se_schema_missing?(diff)
+            puts("Step 1: Presence — processing → #{missing ? 'missing' : 'present'}")
+
+            applied, indexed_inside_apply = __se_full_apply_if_missing(client, missing)
+            drift = __se_full_check_drift(diff, missing, force_rebuild)
+            applied, indexed_inside_apply = __se_full_apply_if_drift(
+              client,
+              drift,
+              applied,
+              indexed_inside_apply,
+              force_rebuild
+            )
+            __se_full_indexation(applied, indexed_inside_apply)
+            __se_full_retention(applied, logical, client)
+          end
+
+          def __se_full_apply_if_missing(client, missing)
+            applied = false
+            indexed_inside_apply = false
+            if missing
+              puts('Step 2: Create+Apply Schema — processing')
+              SearchEngine::Schema.apply!(self, client: client) do |new_physical|
+                indexed_inside_apply = __se_index_partitions!(into: new_physical)
+              end
+              applied = true
+              puts('Step 2: Create+Apply Schema — done')
+            else
+              puts('Step 2: Create+Apply Schema — skip (collection present)')
+            end
+            [applied, indexed_inside_apply]
+          end
+
+          def __se_full_check_drift(diff, missing, force_rebuild)
+            unless missing
+              puts('Step 3: Check Schema Status — processing')
+              drift = __se_schema_drift?(diff)
+              if force_rebuild && !drift
+                puts('Step 3: Check Schema Status — force_rebuild')
+                return true
+              end
+              puts("Step 3: Check Schema Status — #{drift ? 'drift' : 'in_sync'}")
+              return drift
+            end
+            puts('Step 3: Check Schema Status — skip (just created)')
+            false
+          end
+
+          def __se_full_apply_if_drift(client, drift, applied, indexed_inside_apply, force_rebuild)
+            if drift
+              puts('Step 4: Apply New Schema — processing')
+              SearchEngine::Schema.apply!(self, client: client, force_rebuild: force_rebuild) do |new_physical|
+                indexed_inside_apply = __se_index_partitions!(into: new_physical)
+              end
+              applied = true
+              puts('Step 4: Apply New Schema — done')
+            else
+              puts('Step 4: Apply New Schema — skip')
+            end
+            [applied, indexed_inside_apply]
+          end
+
+          def __se_full_indexation(applied, indexed_inside_apply)
+            cascade_ok = false
+            if applied && indexed_inside_apply
+              puts('Step 5: Indexing — skip (performed during schema apply)')
+              begin
+                cascade_ok = indexed_inside_apply.to_sym == :ok
+              rescue StandardError
+                cascade_ok = false
+              end
+            else
+              puts('Step 5: Indexing — processing')
+              idx_status = __se_index_partitions!(into: nil)
+              puts('Step 5: Indexing — done')
+              cascade_ok = (idx_status == :ok)
+            end
+            __se_cascade_after_indexation!(context: :full) if cascade_ok
+          end
+
+          def __se_full_retention(applied, logical, client)
+            if applied
+              puts('Step 6: Retention Cleanup — skip (handled by schema apply)')
+            else
+              puts('Step 6: Retention Cleanup — processing')
+              dropped = __se_retention_cleanup!(logical: logical, client: client)
+              puts("Step 6: Retention Cleanup — dropped=#{dropped.inspect}")
+            end
+          end
+
+          def __se_index_partial(partition:, client:, pre: nil)
+            partitions = Array(partition)
+            diff_res = SearchEngine::Schema.diff(self, client: client)
+            diff = diff_res[:diff] || {}
+
+            missing = __se_schema_missing?(diff)
+            puts("Step 1: Presence — processing → #{missing ? 'missing' : 'present'}")
+            if missing
+              puts('Partial: collection is not present. Quit early.')
+              return
+            end
+
+            puts('Step 2: Check Schema Status — processing')
+            drift = __se_schema_drift?(diff)
+            if drift
+              puts('Partial: schema is not up-to-date. Exit early (run full indexing).')
+              return
+            end
+            puts('Step 2: Check Schema Status — in_sync')
+
+            __se_preflight_dependencies!(mode: pre, client: client) if pre
+
+            puts('Step 3: Partial Indexing — processing')
+            all_ok = true
+            partitions.each do |p|
+              summary = SearchEngine::Indexer.rebuild_partition!(self, partition: p, into: nil)
+              puts(SearchEngine::Logging::PartitionProgress.line(p, summary))
+              begin
+                all_ok &&= (summary.status == :ok)
+              rescue StandardError
+                all_ok &&= false
+              end
+            end
+            puts('Step 3: Partial Indexing — done')
+            __se_cascade_after_indexation!(context: :full) if all_ok
+          end
+
+          # rubocop:disable Metrics/PerceivedComplexity, Metrics/AbcSize
+          def __se_cascade_after_indexation!(context: :full)
+            if SearchEngine::Instrumentation.context&.[](:bulk_suppress_cascade)
+              puts
+              puts('>>>>>> Cascade Referencers — suppressed (bulk)')
+              return
+            end
+            puts
+            puts(%(>>>>>> Cascade Referencers))
+            results = SearchEngine::Cascade.cascade_reindex!(source: self, ids: nil, context: context)
+            outcomes = Array(results[:outcomes])
+            if outcomes.empty?
+              puts('  none')
+            else
+              outcomes.each do |o|
+                coll = o[:collection] || o['collection']
+                mode = (o[:mode] || o['mode']).to_s
+                case mode
+                when 'partial'
+                  puts(%(  Referencer "#{coll}" → partial reindex))
+                when 'full'
+                  puts(%(  Referencer "#{coll}" → full reindex))
+                when 'skipped_unregistered'
+                  puts(%(  Referencer "#{coll}" → skipped (unregistered)))
+                when 'skipped_cycle'
+                  puts(%(  Referencer "#{coll}" → skipped (cycle)))
+                else
+                  puts(%(  Referencer "#{coll}" → #{mode}))
+                end
+              end
+            end
+            puts('>>>>>> Cascade Done')
+          rescue StandardError => error
+            base = "Cascade — error=#{error.class}: #{error.message.to_s[0, 200]}"
+            if error.respond_to?(:status) || error.respond_to?(:body)
+              status = begin
+                error.respond_to?(:status) ? error.status : nil
+              rescue StandardError
+                nil
+              end
+              body_preview = begin
+                b = error.respond_to?(:body) ? error.body : nil
+                if b.is_a?(String)
+                  b[0, 500]
+                elsif b.is_a?(Hash)
+                  b.inspect[0, 500]
+                else
+                  b.to_s[0, 500]
+                end
+              rescue StandardError
+                nil
+              end
+              warn([base, ("status=#{status}" if status), ("body=#{body_preview}" if body_preview)].compact.join(' '))
+            else
+              warn(base)
+            end
+          end
+          # rubocop:enable Metrics/PerceivedComplexity, Metrics/AbcSize
+
+          def __se_retention_cleanup!(*_)
+            SearchEngine::Schema.prune_history!(self)
+          rescue StandardError
+            nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/search_engine/base/index_maintenance/schema.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module SearchEngine
+  class Base
+    module IndexMaintenance
+      # Schema lifecycle helpers (ensure/apply/drop/prune).
+      module Schema
+        extend ActiveSupport::Concern
+
+        class_methods do
+          def schema
+            SearchEngine::Schema.compile(self)
+          end
+
+          def current_schema
+            client = SearchEngine.client
+            logical = respond_to?(:collection) ? collection.to_s : name.to_s
+            physical = client.resolve_alias(logical) || logical
+            client.retrieve_collection_schema(physical)
+          end
+
+          def schema_diff
+            client = SearchEngine.client
+            res = SearchEngine::Schema.diff(self, client: client)
+            res[:diff]
+          end
+
+          def update_collection!
+            client = SearchEngine.client
+
+            puts 'Update Collection — analyzing diff for in-place update...'
+            updated = SearchEngine::Schema.update!(self, client: client)
+
+            if updated
+              puts 'Update Collection — schema updated in-place (PATCH)'
+            else
+              puts 'Update Collection — in-place update not possible (no changes or incompatible)'
+            end
+            updated
+          end
+
+          def drop_collection!
+            client = SearchEngine.client
+            logical = respond_to?(:collection) ? collection.to_s : name.to_s
+
+            # Resolve alias with a safer timeout for control-plane operations
+            alias_target = client.resolve_alias(logical, timeout_ms: 10_000)
+            physical = if alias_target && !alias_target.to_s.strip.empty?
+                         alias_target.to_s
+                       else
+                         live = client.retrieve_collection_schema(logical, timeout_ms: 10_000)
+                         live ? logical : nil
+                       end
+
+            if physical.nil?
+              puts('Drop Collection — skip (not present)')
+              return
+            end
+
+            puts
+            puts(%(>>>>>> Dropping Collection "#{logical}"))
+            puts("Drop Collection — processing (logical=#{logical} physical=#{physical})")
+            # Use an extended timeout to accommodate large collection drops
+            client.delete_collection(physical, timeout_ms: 60_000)
+            puts('Drop Collection — done')
+            puts(%(>>>>>> Dropped Collection "#{logical}"))
+            nil
+          end
+
+          def recreate_collection!
+            client = SearchEngine.client
+            logical = respond_to?(:collection) ? collection.to_s : name.to_s
+
+            alias_target = client.resolve_alias(logical)
+            physical = if alias_target && !alias_target.to_s.strip.empty?
+                         alias_target.to_s
+                       else
+                         live = client.retrieve_collection_schema(logical)
+                         live ? logical : nil
+                       end
+
+            if physical
+              puts("Recreate Collection — dropping existing (logical=#{logical} physical=#{physical})")
+              client.delete_collection(physical)
+            else
+              puts('Recreate Collection — no existing collection (skip drop)')
+            end
+
+            schema = SearchEngine::Schema.compile(self)
+            puts("Recreate Collection — creating collection with schema (logical=#{logical})")
+            client.create_collection(schema)
+            puts('Recreate Collection — done')
+            nil
+          end
+
+          def __se_retention_cleanup!(_logical:, _client:)
+            SearchEngine::Schema.prune_history!(self)
+          end
+
+          def __se_schema_missing?(diff)
+            opts = diff[:collection_options]
+            opts.is_a?(Hash) && opts[:live] == :missing
+          end
+
+          def __se_schema_drift?(diff)
+            added = Array(diff[:added_fields])
+            removed = Array(diff[:removed_fields])
+            changed = (diff[:changed_fields] || {}).to_h
+            coll_opts = (diff[:collection_options] || {}).to_h
+            stale_refs = Array(diff[:stale_references])
+            added.any? || removed.any? || !changed.empty? || !coll_opts.empty? || stale_refs.any?
+          end
+        end
+      end
+    end
+  end
+end