caoutsearch 0.0.0

data/lib/caoutsearch/index/indice.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Caoutsearch
+  module Index
+    module Indice
+      extend ActiveSupport::Concern
+
+      included do
+        delegate :refresh_indice, to: :class
+      end
+
+      class_methods do
+        # Create index or an alias
+        #
+        #   ArticleIndex.create_indice
+        #
+        def create_indice
+          client.indices.create(
+            index: index_name,
+            body: {
+              settings: settings.as_json,
+              mappings: mappings.as_json
+            }
+          )
+        end
+
+        # Verify index existence
+        #
+        #   ArticleIndex.indice_exists?
+        #   => true
+        #
+        def indice_exists?
+          client.indices.exists?(index: index_name)
+        end
+
+        # Verify index existence
+        #
+        #   ArticleIndex.delete_indice
+        #
+        def delete_indice
+          client.indices.delete(index: index_name)
+        end
+
+        # Explicitly refresh one or more index, making all operations performed
+        # since the last refresh available for search.
+        #
+        #   ArticleIndex.refresh_indice
+        #
+        def refresh_indice
+          client.indices.refresh(index: index_name)
+        end
+      end
+    end
+  end
+end

data/lib/caoutsearch/index/indice_versions.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+module Caoutsearch
+  module Index
+    module IndiceVersions
+      extend ActiveSupport::Concern
+
+      class_methods do
+        # List versions
+        #
+        #   ArticleIndex.indice_versions
+        #   => ["articles_v0", "articles_v1"]
+        #
+        def indice_versions
+          client.cat
+            .indices(h: ["index"], format: :json)
+            .filter_map { |h| h["index"] }
+            .grep(indice_version_regexp)
+        end
+
+        # List aliased versions
+        #
+        #   ArticleIndex.aliased_indice_versions
+        #   => ["articles_v1"]
+        #
+        def aliased_indice_versions
+          client.indices
+            .get_alias(name: index_name)
+            .keys
+            .grep(indice_version_regexp)
+        rescue Elastic::Transport::Transport::Errors::NotFound
+          []
+        end
+
+        # List last version available (aliased or not)
+        #
+        #   ArticleIndex.last_indice_version
+        #   => "articles_v1"
+        #
+        def last_indice_version
+          indice_versions.max
+        end
+
+        # Next version to create
+        #
+        #   ArticleIndex.next_indice_version
+        #   => "articles_v2"
+        #
+        def next_indice_version
+          current_version = last_indice_version
+
+          if current_version
+            number = current_version[/_v(\d+)$/, 1].to_i
+            "#{index_name}_v#{number + 1}"
+          else
+            "#{index_name}_v0"
+          end
+        end
+
+        # Create new version of the index
+        #
+        #   ArticleIndex.create_indice_version
+        #   => "articles_v2"
+        #
+        def create_indice_version(version_name = next_indice_version)
+          client.indices.create(
+            index: version_name,
+            body: {
+              settings: settings.as_json,
+              mappings: mappings.as_json
+            }
+          )
+
+          version_name
+        end
+
+        # Switch the index to a new version
+        #
+        #   ArticleIndex.switch_indice_version("articles_v2")
+        #   => true
+        #
+        def switch_indice_version(version_name = :__last__)
+          version_name = last_indice_version if version_name == :__last__
+
+          actions = []
+          actions << { add: { index: version_name, alias: index_name } }
+
+          aliased_indice_versions.each do |alias_name|
+            return false if alias_name == version_name
+            actions << { remove: { index: alias_name, alias: index_name } }
+          end
+
+          client.indices.update_aliases(body: { actions: actions })
+          refresh_indice
+          version_name
+        end
+
+        # Prune not-aliased versions
+        #
+        def prune_indice_versions
+          old_versions = indice_versions - aliased_indice_versions
+          old_versions.each do |version_name|
+            client.indices.delete(index: version_name)
+          end
+        end
+
+        # Delete all versions, alias or not
+        #
+        def delete_all_indice_versions
+          indice_versions.each do |version_name|
+            client.indices.delete(index: version_name)
+          end
+        end
+
+        private
+
+        def indice_version_regexp
+          @version_regexp ||= /^#{index_name}_(v\d+)$/
+        end
+      end
+    end
+  end
+end

data/lib/caoutsearch/index/instrumentation.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Caoutsearch
+  module Index
+    module Instrumentation
+      extend ActiveSupport::Concern
+
+      def instrument(action, **options, &block)
+        ActiveSupport::Notifications.instrument("#{action}.caoutsearch_index", **options, klass: self.class.to_s, &block)
+      end
+
+      class_methods do
+        def instrument(action, **options, &block)
+          ActiveSupport::Notifications.instrument("#{action}.caoutsearch_index", **options, klass: to_s, &block)
+        end
+      end
+    end
+  end
+end

data/lib/caoutsearch/index/internal_dsl.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module Caoutsearch
+  module Index
+    module InternalDSL
+      extend ActiveSupport::Concern
+
+      included do
+        # Be careful with these class attributes
+        # Always use `+=` or `.dup.merge` to assign a new copy
+        #
+        class_attribute :properties,            instance_writer: false, default: []
+        class_attribute :partial_reindexations, instance_writer: false, default: {}
+      end
+
+      class_methods do
+        # Declare a property
+        #
+        #   class ArticleIndex < Caoutsearch::Index::Base
+        #     property :title
+        #     property :description
+        #     property :tags
+        #
+        #     def tags
+        #       record.tags.map do |tag|
+        #         {
+        #           label: tag.label,
+        #           score: tag.score
+        #         }
+        #        end
+        #     end
+        #   end
+        #
+        def property(key, body = nil)
+          raise ArgumentError, "The property body needs to be callable." if body && !body.respond_to?(:call)
+
+          key = key.to_s
+          self.properties += [key] unless properties.include?(key)
+
+          define_method(key, &body) if body
+        end
+
+        # Declare an alias-property for partial reindexation
+        #
+        #   class LocalIndex < Caoutsearch::Index::Base
+        #     property :invariant
+        #     property :geoaddress
+        #     property :geoposition
+        #
+        #     allow_partial_reindex :post_processed_data, properties: %i[geoaddress occupation]
+        #   end
+        #
+        def allow_partial_reindex(name, body = nil, properties: nil, upsert: false)
+          raise ArgumentError, "The allow_partial_reindex body needs to be callable." if body && !body.respond_to?(:call)
+
+          name = name.to_s
+          self.partial_reindexations = partial_reindexations.dup.merge(name => { properties: properties })
+
+          if body
+            define_method(name, &body)
+          else
+            define_method(name) do
+              body = { doc: properties.index_with { |key| send(key) } }
+              body[:doc_as_upsert] = true if upsert
+              body
+            end
+          end
+        end
+
+        def allow_reindex?(subject)
+          subject = subject.to_s
+          properties.include?(subject) || partial_reindexations.include?(subject)
+        end
+      end
+    end
+  end
+end

data/lib/caoutsearch/index/naming.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Caoutsearch
+  module Index
+    module Naming
+      extend ActiveSupport::Concern
+
+      included do
+        delegate :index_name, to: :class
+      end
+
+      class_methods do
+        def index_name
+          @index_name ||= default_index_name
+        end
+
+        def index_name=(name)
+          @index_name = name
+        end
+
+        private
+
+        def default_index_name
+          name.gsub(/Index$/, "").tableize.tr("/", "_")
+        end
+      end
+    end
+  end
+end

data/lib/caoutsearch/index/reindex.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module Caoutsearch
+  module Index
+    module Reindex
+      extend ActiveSupport::Concern
+
+      class_methods do
+        # Reindex multiple records with bulk API
+        #
+        # Examples:
+        #     ArticleIndex.reindex(Article.all)
+        #     ArticleIndex.reindex(Article.modified_since(2.days), :title, :content)
+        #
+        # Examples with options:
+        #
+        #     articles = Article.modified_since(2.days)
+        #
+        #     ArticleIndex.reindex(articles, index: "article_v3")
+        #     ArticleIndex.reindex(articles, batch_size: 10)
+        #     ArticleIndex.reindex(articles, method: :update)
+        #     ArticleIndex.reindex(articles, method: :update)
+        #
+        # When passing a limited set of records (< 100), you can pass a `limited_set` option to avoid
+        # extra queries
+        #     ArticleIndex.reindex(Article.limit(100), limited_set: true)
+        #
+        def reindex(records, *keys, **options)
+          options.assert_valid_keys(:index, :refresh, :batch_size, :method, :total, :progress)
+          keys.flatten!
+
+          records = apply_scopes(records, keys)
+          records = records.strict_loading
+
+          index            = options.fetch(:index, index_name)
+          refresh          = options.fetch(:refresh, false)
+          method           = options.fetch(:method) { keys.present? ? :update : :index }
+          batch_size       = options[:batch_size] || 100
+          total            = options[:total] || records.count(:all)
+          progress         = options[:progress]
+          current_progress = 0
+
+          return if total.zero?
+
+          progress&.total    = total
+          progress&.progress = current_progress
+
+          finder = if total <= batch_size
+            records.to_a.each_slice(total)
+          else
+            records.find_in_batches(batch_size: batch_size)
+          end
+
+          finder.each do |batch|
+            current_progress += batch.size
+            request_payload   = {
+              index:  index,
+              body:   bulkify(batch, method, keys)
+            }
+
+            instrument(:reindex, total: total, progress: current_progress, records: batch) do |event_payload|
+              event_payload[:request]  = request_payload
+              event_payload[:response] = client.bulk(request_payload)
+            end
+
+            progress&.increment(batch.size)
+            records.connection.clear_query_cache
+          end
+
+          refresh_indice(index: index) if refresh
+        end
+
+        alias_method :update_documents, :reindex
+      end
+    end
+  end
+end

data/lib/caoutsearch/index/scoping.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Caoutsearch
+  module Index
+    module Scoping
+      extend ActiveSupport::Concern
+
+      included do
+        class_attribute :scopes,   instance_accessor: false, default: {}
+        class_attribute :preloads, instance_accessor: false, default: {}
+      end
+
+      class_methods do
+        def default_scope(body)
+          scope :_default, body
+        end
+
+        def scope(name, body)
+          raise ArgumentError, "The scope body needs to be callable." unless body.respond_to?(:call)
+
+          name = name.to_s
+          self.scopes = scopes.dup.merge(name => body)
+        end
+
+        def preload(name, with: nil)
+          name = name.to_s
+          with = Array.wrap(with || name)
+
+          scope name, ->(records) { records.preload(*with) }
+        end
+
+        private
+
+        def apply_scopes(records, names = [])
+          names = names.map(&:to_s)
+          names = properties if names.empty?
+          names += %w[_default] # Use += instead of << to create a copy
+
+          names.each do |name|
+            if scopes.include?[name]
+              scope   = scopes[name]
+              records = scope.call(records)
+            elsif partial_reindexations.include?(name)
+              properties = partial_reindexations.dig(name, :properties)
+              records    = apply_scopes(records, properties) if properties&.any?
+            end
+          end
+
+          records
+        end
+      end
+    end
+  end
+end

data/lib/caoutsearch/index/serialization.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+module Caoutsearch
+  module Index
+    module Serialization
+      extend ActiveSupport::Concern
+
+      included do
+        delegate :to_json, to: :as_json
+      end
+
+      class_methods do
+        # Transform record or array of records to JSON:
+        #
+        #    transform(nil)               => nil
+        #    transform(record.first)      => { ... }
+        #    transform(record.limit(100)) => [{ ... }, { ... }, ...]
+        #
+        def transform(input, *keys)
+          if input.nil?
+            nil
+          elsif input.respond_to?(:map)
+            input.map { |record| transform(record, *keys) }
+          else
+            new(input).as_json(*keys)
+          end
+        end
+
+        # Convert an array of records to an Elasticsearch bulk payload
+        #
+        def bulkify(input, method, keys)
+          raise ArgumentError, "unknown method #{method}" unless %i[index update delete].include?(method)
+
+          input.reduce([]) do |payload, record|
+            payload + new(record).bulkify(method, keys)
+          end
+        end
+      end
+
+      # Serialize the object payload
+      #
+      def as_json(*keys)
+        keys = keys.map(&:to_s)
+        _, partial_keys = analyze_keys(keys)
+
+        raise SerializationError, format_keys("cannot serializer the following keys together: %{keys}", partial_keys) if keys.size > 1 && partial_keys.any?
+
+        json = {}
+        keys = properties if keys.empty?
+        keys.each do |key|
+          result = send(key)
+
+          if partial_reindexations.include?(key)
+            json = json.merge(result)
+          else
+            json[key.to_sym] = result
+          end
+        end
+
+        simplify(json)
+      end
+
+      # Recursive objects simplication:
+      #
+      #   [nil, 'A', 'A', 'B']          => ['A', 'B']
+      #   [nil, 'A', 'A']               => 'A'
+      #   [nil, nil]                    => nil
+      #   []                            => nil
+      #
+      #   { key: [nil, 'A', 'A', 'B'] } => { key: ['A', 'B'] }
+      #   { key: [nil, 'A', 'A'] }      => { key: 'A' }
+      #   { key: [nil, nil] }           => { key: nil }
+      #   { key: [] }                   => { key: nil }
+      #   { }                           => { }
+      #
+      def simplify(object)
+        case object
+        when Array
+          object = object.filter_map { |array_item| simplify(array_item) }.uniq
+          object = object[0] if object.size <= 1
+        when Hash
+          object.each { |key, value| object[key] = simplify(value) }
+        end
+
+        object
+      end
+
+      # Convert the object Elasticsearch `header\ndata` payload format
+      #
+      def bulkify(method, keys)
+        raise ArgumentError, "unknown method #{method}" unless %i[index update delete].include?(method)
+
+        keys                        = keys.map(&:to_s)
+        payload                     = []
+        property_keys, partial_keys = analyze_keys(keys)
+
+        case method
+        when :index
+          raise SerializationError, format("cannot serialize the following keys: %{keys}", keys: partial_keys.to_sentence) if partial_keys.any?
+
+          payload << { index: { _id: record.id } }
+          payload << as_json(*keys)
+
+        when :update
+          if property_keys.any?
+            payload << { update: { _id: record.id } }
+            payload << { doc: as_json(*property_keys) }
+          end
+
+          partial_keys.each do |key|
+            payload << { update: { _id: record.id } }
+            payload << as_json(*key)
+          end
+
+        when :delete
+          payload << { update: { _id: record.id } }
+        end
+
+        payload
+      end
+
+      private
+
+      def analyze_keys(keys)
+        partial_keys  = partial_reindexations.keys & keys
+        property_keys = properties & keys
+        unknown_keys  = keys - property_keys - partial_keys
+
+        raise ArgumentError, format("unknown keys: %{keys}", keys: unknown_keys.to_sentence) if unknown_keys.any?
+
+        property_keys = properties & keys
+        [property_keys, partial_keys]
+      end
+    end
+  end
+end

data/lib/caoutsearch/index.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Caoutsearch
+  module Index
+    SerializationError = Class.new(StandardError)
+  end
+end

data/lib/caoutsearch/instrumentation/base.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Caoutsearch
+  module Instrumentation
+    class Base < ActiveSupport::LogSubscriber
+      private
+
+      def log_request(subject, event, format: nil)
+        return unless format
+
+        payload  = event.payload
+        request  = payload[:request]
+
+        debug do
+          title        = color("#{payload[:klass]} #{subject}", GREEN, true)
+          request_body = format_request_body(request, format: format)
+
+          message = "  #{title} #{request_body}"
+          message = yield(message, payload) if block_given?
+          message
+        end
+      end
+
+      def log_response(subject, event, warn_errors: false)
+        payload  = event.payload
+        response = payload[:response]
+        return unless response
+
+        debug do
+          title     = color("#{payload[:klass]} #{subject}", GREEN, true)
+
+          duration  = "#{event.duration.round(1)}ms"
+          duration += " / took #{response["took"]}ms" if response.key?("took")
+          duration  = color("(#{duration})", GREEN, true)
+
+          message  = "  #{title} #{duration}"
+          message += " got errors" if response["errors"]
+          message  = yield(message, payload) if block_given?
+
+          message
+        end
+
+        return unless response["errors"] && warn_errors
+
+        errors = response["items"].select { |k, _| k.values.first["error"] }
+        errors.each do |error|
+          warn { color(error, RED, true) }
+        end
+      end
+
+      def format_request_body(body, format: nil)
+        case format
+        when "amazing_print", "awesome_print"
+          body.ai(limit: true, index: false)
+        when "full"
+          json = JSON.dump(body)
+          color(json, BLUE, true)
+        when "truncated"
+          json = JSON.dump(body).truncate(200, omission: "…}")
+          color(json, BLUE, true)
+        end
+      end
+
+      def inspect_json_size(json)
+        ApplicationController.helpers.number_to_human_size(JSON.dump(json).bytesize)
+      end
+    end
+  end
+end