noiseless 0.0.0 → 0.1.0

data/lib/noiseless/ast/root.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Noiseless
+  module AST
+    class Root < Node
+      attr_reader :indexes, :bool, :sort, :paginate, :vector, :collapse, :search_after,
+                  :aggregations, :hybrid, :pipeline, :image_query, :conversation, :joins,
+                  :remove_duplicates, :facet_sample_slope, :pinned_hits
+
+      def initialize(indexes:, bool:, sort:, paginate:, vector: nil, collapse: nil, search_after: nil,
+                     aggregations: [], hybrid: nil, pipeline: nil, image_query: nil, conversation: nil, joins: [],
+                     remove_duplicates: nil, facet_sample_slope: nil, pinned_hits: nil)
+        super()
+        @indexes      = Array(indexes)
+        @bool         = bool
+        @sort         = sort
+        @paginate     = paginate
+        @vector       = vector
+        @collapse     = collapse
+        @search_after = search_after
+        @aggregations = aggregations
+        @hybrid       = hybrid
+        @pipeline     = pipeline
+        @image_query  = image_query
+        @conversation = conversation
+        @joins        = joins
+        @remove_duplicates = remove_duplicates
+        @facet_sample_slope = facet_sample_slope
+        @pinned_hits = pinned_hits
+      end
+
+      def vector_search?
+        !@vector.nil?
+      end
+
+      def hybrid_search?
+        !@hybrid.nil?
+      end
+
+      def has_pipeline?
+        !@pipeline.nil?
+      end
+
+      def collapsed?
+        !@collapse.nil?
+      end
+
+      def cursor_pagination?
+        !@search_after.nil?
+      end
+
+      def aggregated?
+        @aggregations.any?
+      end
+
+      def image_search?
+        !@image_query.nil?
+      end
+
+      def conversational?
+        !@conversation.nil?
+      end
+
+      def has_joins?
+        @joins.any?
+      end
+    end
+  end
+end

data/lib/noiseless/ast/search_after.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Noiseless
+  module AST
+    class SearchAfter < Node
+      attr_reader :values
+
+      def initialize(values)
+        super()
+        @values = Array(values)
+      end
+    end
+  end
+end

data/lib/noiseless/ast/sort.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Noiseless
+  module AST
+    class Sort < Node
+      attr_reader :field, :direction
+
+      def initialize(field, direction)
+        super()
+        @field = field
+        @direction = direction
+      end
+    end
+  end
+end

data/lib/noiseless/ast/vector.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Noiseless
+  module AST
+    # Vector search node for semantic/embedding-based search
+    # Used with pgvector in PostgreSQL or knn in OpenSearch
+    class Vector < Node
+      attr_reader :field, :embedding, :k, :distance_metric
+
+      # @param field [Symbol, String] The embedding column/field
+      # @param embedding [Array<Float>] The query embedding vector
+      # @param k [Integer] Number of nearest neighbors (default: 10)
+      # @param distance_metric [Symbol] :cosine, :l2, or :inner_product (default: :cosine)
+      def initialize(field, embedding, k: 10, distance_metric: :cosine)
+        super()
+        @field = field
+        @embedding = embedding
+        @k = k
+        @distance_metric = distance_metric
+      end
+
+      def dimension
+        @embedding&.size || 0
+      end
+    end
+  end
+end

data/lib/noiseless/ast/wildcard.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Noiseless
+  module AST
+    class Wildcard < Node
+      attr_reader :field, :value
+
+      def initialize(field, value)
+        super()
+        @field = field
+        @value = value
+      end
+    end
+  end
+end

data/lib/noiseless/ast.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Noiseless
+  module AST
+    class Node
+      def to_h
+        hash = {}
+        instance_variables.each do |var|
+          key = var.to_s.delete("@").to_sym
+          value = instance_variable_get(var)
+
+          hash[key] = case value
+                      when Node
+                        value.to_h
+                      when Array
+                        value.map { |item| item.is_a?(Node) ? item.to_h : item }
+                      else
+                        value
+                      end
+        end
+
+        # Include the class name for better introspection
+        hash[:_type] = self.class.name.split("::").last
+        hash
+      end
+
+      alias to_hash to_h
+    end
+  end
+end

data/lib/noiseless/bulk_importer.rb

@@ -0,0 +1,195 @@
+# frozen_string_literal: true
+
+module Noiseless
+  class BulkImporter
+    attr_reader :model_class, :errors
+
+    def initialize(model_class, connection: nil)
+      @model_class = model_class
+      @connection = connection || model_class.connection
+      @errors = []
+    end
+
+    def import(relation_or_records = nil,
+               batch_size: 1000,
+               transform: nil,
+               preprocess: nil,
+               force: false,
+               refresh: true,
+               **)
+      @errors.clear
+
+      # Create index if force is true
+      if force
+        delete_index
+        create_index
+      end
+
+      # Get records to import
+      records = resolve_records(relation_or_records)
+
+      total_imported = 0
+
+      records.each_slice(batch_size) do |batch|
+        # Apply preprocessing to the entire batch
+        processed_batch = preprocess ? preprocess.call(batch) : batch
+
+        # Transform individual records and build actions
+        actions = build_bulk_actions(processed_batch, transform)
+
+        # Execute bulk operation
+        begin
+          client = Noiseless.connections.client(@connection)
+          response = client.bulk(actions, refresh: refresh, **)
+
+          # Check for errors in response
+          collect_errors(response, processed_batch)
+
+          total_imported += actions.size
+        rescue StandardError => e
+          @errors << {
+            error: e.message,
+            batch: processed_batch.map { |r| identify_record(r) }
+          }
+        end
+      end
+
+      {
+        imported: total_imported,
+        errors: @errors.size,
+        error_details: @errors
+      }
+    end
+
+    def import_scoped(scope, **)
+      import(scope, **)
+    end
+
+    def reindex(batch_size: 1000, **)
+      raise ArgumentError, "Model class #{model_class} must respond to :all for reindexing" unless model_class.respond_to?(:all)
+
+      import(model_class.all, batch_size: batch_size, force: true, **)
+    end
+
+    private
+
+    def resolve_records(relation_or_records)
+      case relation_or_records
+      when nil
+        model_class.respond_to?(:all) ? model_class.all : []
+      when String, Symbol
+        # Assume it's a scope name
+        if model_class.respond_to?(relation_or_records)
+          model_class.public_send(relation_or_records)
+        else
+          []
+        end
+      else
+        relation_or_records
+      end
+    end
+
+    def build_bulk_actions(batch, transform)
+      batch.filter_map do |record|
+        # Apply transform function if provided
+        document = transform ? transform.call(record) : default_transform(record)
+        next unless document
+
+        {
+          index: {
+            _index: index_name,
+            _id: extract_id(record),
+            data: document
+          }
+        }
+      rescue StandardError => e
+        @errors << {
+          error: e.message,
+          record: identify_record(record)
+        }
+        nil
+      end
+    end
+
+    def default_transform(record)
+      if record.respond_to?(:to_h)
+        record.to_h
+      elsif record.respond_to?(:attributes)
+        record.attributes
+      else
+        record
+      end
+    end
+
+    def extract_id(record)
+      if record.respond_to?(:id)
+        record.id
+      elsif record.is_a?(Hash)
+        record[:id] || record["id"]
+      else
+        record.object_id
+      end
+    end
+
+    def identify_record(record)
+      id = extract_id(record)
+      {
+        id: id,
+        class: record.class.name,
+        object_id: record.object_id
+      }
+    end
+
+    def collect_errors(response, batch)
+      return unless response.is_a?(Hash) && response["items"]
+
+      response["items"].each_with_index do |item, index|
+        action = item.keys.first
+        result = item[action]
+
+        next unless result["error"]
+
+        record = batch[index]
+        @errors << {
+          error: result["error"],
+          record: identify_record(record),
+          status: result["status"]
+        }
+      end
+    end
+
+    def index_name
+      @index_name ||= if model_class.respond_to?(:search_index)
+                        Array(model_class.search_index).first
+                      else
+                        model_class.name.demodulize.underscore.pluralize
+                      end
+    end
+
+    def delete_index
+      client = Noiseless.connections.client(@connection)
+      client.delete_index(index_name)
+    rescue StandardError => _e
+      # Index might not exist, which is fine
+      nil
+    end
+
+    def create_index
+      return unless model_class.respond_to?(:mapping)
+
+      mapping_block = model_class.mapping
+      return unless mapping_block
+
+      begin
+        _client = Noiseless.connections.client(@connection)
+        # This would need to be implemented in the adapter
+        # client.create_index(index_name, mapping: mapping_block)
+      rescue StandardError => e
+        @errors << {
+          error: "Failed to create index: #{e.message}",
+          index: index_name
+        }
+      end
+    end
+  end
+end

data/lib/noiseless/callbacks.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+require "active_support/concern"
+
+module Noiseless
+  module Callbacks
+    extend ActiveSupport::Concern
+
+    included do
+      after_save :update_search_index_on_save
+      after_destroy :remove_from_search_index
+      after_commit :update_search_index_on_commit, on: %i[create update]
+      after_commit :remove_from_search_index_on_commit, on: :destroy
+    end
+
+    class_methods do
+      def auto_index(enabled: true, **options)
+        @auto_index_enabled = enabled
+        @auto_index_options = options
+      end
+
+      def auto_index_enabled?
+        @auto_index_enabled != false
+      end
+
+      def auto_index_options
+        @auto_index_options ||= {}
+      end
+
+      def skip_auto_index
+        previous_value = Thread.current[:noiseless_skip_auto_index]
+        Thread.current[:noiseless_skip_auto_index] = true
+        yield
+      ensure
+        Thread.current[:noiseless_skip_auto_index] = previous_value
+      end
+    end
+
+    private
+
+    def should_update_search_index?
+      return false if Thread.current[:noiseless_skip_auto_index]
+      return false unless self.class.auto_index_enabled?
+
+      # Only update if we have searchable content
+      if respond_to?(:searchable?)
+        searchable?
+      else
+        true
+      end
+    end
+
+    def update_search_index_on_save
+      return unless should_update_search_index?
+
+      update_search_index_async if noiseless_new_record? || (respond_to?(:changed?) && changed?)
+    rescue Net::ProtocolError, JSON::ParserError, Timeout::Error => e
+      handle_search_index_error(e, :update)
+    end
+
+    def update_search_index_on_commit
+      return unless should_update_search_index?
+
+      update_search_index_async
+    rescue Net::ProtocolError, JSON::ParserError, Timeout::Error => e
+      handle_search_index_error(e, :update)
+    end
+
+    def remove_from_search_index
+      return unless should_update_search_index?
+
+      remove_from_search_index_async
+    rescue Net::ProtocolError, JSON::ParserError, Timeout::Error => e
+      handle_search_index_error(e, :delete)
+    end
+
+    def remove_from_search_index_on_commit
+      return unless should_update_search_index?
+
+      remove_from_search_index_async
+    rescue Net::ProtocolError, JSON::ParserError, Timeout::Error => e
+      handle_search_index_error(e, :delete)
+    end
+
+    def update_search_index_async
+      options = self.class.auto_index_options
+
+      if options[:async]
+        # Queue for background processing
+        SearchIndexUpdateJob.perform_later(
+          self.class.name,
+          id,
+          "update",
+          options
+        )
+      else
+        # Immediate update
+        document_manager.update_document(**options)
+      end
+    end
+
+    def remove_from_search_index_async
+      options = self.class.auto_index_options
+
+      if options[:async]
+        # Queue for background processing
+        SearchIndexUpdateJob.perform_later(
+          self.class.name,
+          id,
+          "delete",
+          options
+        )
+      else
+        # Immediate removal
+        document_manager.delete_document(**options)
+      end
+    end
+
+    def handle_search_index_error(error, operation)
+      options = self.class.auto_index_options
+
+      if options[:raise_on_error]
+        raise error
+      elsif (logger = Rails.logger)
+        # Log the error or handle silently based on configuration
+        logger.error "Noiseless: Failed to #{operation} search index for #{self.class.name}##{id}: #{error.message}"
+      end
+    end
+
+    def noiseless_new_record?
+      if respond_to?(:persisted?)
+        !persisted?
+      else
+        !id
+      end
+    end
+  end
+end

data/lib/noiseless/connection_manager.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Noiseless
+  class ConnectionManager
+    def initialize
+      @clients = {}
+      @configs = {}
+    end
+
+    # Register a named client statically from YAML (boot-time only)
+    def register(name, adapter:, hosts:)
+      @configs[name.to_sym] = { adapter: adapter, hosts: hosts }
+    end
+
+    # Retrieve a client; defaults to :primary
+    def client(name = :primary)
+      name = name.to_sym
+
+      # Lazy-load the adapter only when actually used
+      @clients[name] ||= begin
+        config = @configs.fetch(name) { raise "Unknown connection: #{name}" }
+        Adapters.lookup(config[:adapter], hosts: config[:hosts])
+      end
+    end
+  end
+end

data/lib/noiseless/document_manager.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+module Noiseless
+  class DocumentManager
+    def initialize(model_instance, connection: nil)
+      @model_instance = model_instance
+      @connection = connection || model_instance.class.connection
+    end
+
+    def index_document(refresh: false, **)
+      document = build_document
+      return false unless document
+
+      client = Noiseless.connections.client(@connection)
+      client.index_document(
+        index: index_name,
+        id: document_id,
+        document: document,
+        refresh: refresh,
+        **
+      )
+    end
+
+    def update_document(refresh: false, detect_changes: true, **)
+      if detect_changes && supports_dirty_tracking?
+        return false unless has_changes?
+
+        changes = extract_changes
+        return false if changes.empty?
+
+        client = Noiseless.connections.client(@connection)
+        client.update_document(
+          index: index_name,
+          id: document_id,
+          changes: changes,
+          refresh: refresh,
+          **
+        )
+      else
+        # Fall back to full document update
+        index_document(refresh: refresh, **)
+      end
+    end
+
+    def delete_document(refresh: false, **)
+      client = Noiseless.connections.client(@connection)
+      client.delete_document(
+        index: index_name,
+        id: document_id,
+        refresh: refresh,
+        **
+      )
+    end
+
+    def document_exists?
+      client = Noiseless.connections.client(@connection)
+      client.document_exists?(
+        index: index_name,
+        id: document_id
+      )
+    end
+
+    private
+
+    attr_reader :model_instance
+
+    def build_document
+      if model_instance.respond_to?(:to_search_document)
+        model_instance.to_search_document
+      elsif model_instance.respond_to?(:to_h)
+        model_instance.to_h
+      elsif model_instance.respond_to?(:attributes)
+        model_instance.attributes
+      end
+    end
+
+    def document_id
+      if model_instance.respond_to?(:id)
+        model_instance.id
+      elsif model_instance.respond_to?(:[])
+        model_instance[:id] || model_instance["id"]
+      else
+        model_instance.object_id
+      end
+    end
+
+    def index_name
+      @index_name ||= if model_instance.class.respond_to?(:search_index)
+                        Array(model_instance.class.search_index).first
+                      else
+                        model_instance.class.name.demodulize.underscore.pluralize
+                      end
+    end
+
+    def supports_dirty_tracking?
+      model_instance.respond_to?(:changed_attributes) ||
+        model_instance.respond_to?(:changes) ||
+        model_instance.respond_to?(:changed?)
+    end
+
+    def has_changes?
+      return true unless supports_dirty_tracking?
+
+      if model_instance.respond_to?(:changed?)
+        model_instance.changed?
+      elsif model_instance.respond_to?(:changes)
+        !model_instance.changes.empty?
+      elsif model_instance.respond_to?(:changed_attributes)
+        !model_instance.changed_attributes.empty?
+      else
+        true
+      end
+    end
+
+    def extract_changes
+      changes = {}
+
+      if model_instance.respond_to?(:changes)
+        # ActiveModel::Dirty style changes hash
+        model_instance.changes.each do |attr, (_old_value, new_value)|
+          changes[attr] = new_value
+        end
+      elsif model_instance.respond_to?(:changed_attributes)
+        # Get current values for changed attributes
+        model_instance.changed_attributes.each_key do |attr|
+          if model_instance.respond_to?(attr)
+            changes[attr] = model_instance.public_send(attr)
+          elsif model_instance.respond_to?(:[])
+            changes[attr] = model_instance[attr]
+          end
+        end
+      end
+
+      changes
+    end
+  end
+end