stretchy-model 0.6.0 → 0.6.5

data/lib/opensearch/api/actions/machine_learning/models/undeploy.rb

@@ -0,0 +1,31 @@
+module OpenSearch
+  module API
+    module MachineLearning
+	    module Models
+	      module Actions
+	        # Register a model.
+	        #
+	        # @option arguments [String] :id The model id
+	        # @option arguments [Hash] :body The deploy options
+	        #
+          # 
+          # POST /_plugins/_ml/models/<model_id>/_undeploy
+	        def undeploy(arguments = {})
+						raise ArgumentError, "Required argument 'id' missing" unless arguments[:id]
+	          _id = arguments.delete(:id)
+	          arguments = arguments.clone
+            headers = arguments.delete(:headers) || {}
+	
+	          method = OpenSearch::API::HTTP_POST
+	          path   = "_plugins/_ml/models/#{Utils.__listify(_id)}/_undeploy"
+	          params = Utils.__validate_and_extract_params arguments, ParamsRegistry.get(__method__)
+	
+	          body = arguments[:body]
+	          perform_request(method, path, params, body, headers).body
+	        end
+	
+	      end
+	    end
+    end
+  end
+end

data/lib/opensearch/api/actions/machine_learning/models/update_model.rb

@@ -0,0 +1,39 @@
+module OpenSearch
+  module API
+    module MachineLearning
+	    module Models
+	      module Actions
+	        # Returns a model.
+	        #
+	        # @option arguments [String] :id The model id
+	        # @option arguments [Hash] :body The request fields
+	        #
+	        # Example
+	        # update_model(id: 109sdj0asl092, "rate_limiter": {
+          #    "limit": "4",
+          #    "unit": "MINUTES"
+          #  }
+          # )
+          # 
+          # PUT /_plugins/_ml/models/<model_id>
+	        def update_model(arguments = {})
+		        raise ArgumentError, "Required argument 'body' missing" unless arguments[:body]
+		        raise ArgumentError, "Required argument 'id' missing" unless arguments[:id]
+		        
+            _id = arguments.delete(:id)
+
+            headers = arguments.delete(:headers) || {}
+
+		        method = OpenSearch::API::HTTP_PUT
+		        path   = "_plugins/_ml/models/#{Utils.__listify(_id)}"
+		        params = Utils.__validate_and_extract_params arguments, ParamsRegistry.get(__method__)
+				
+            body = arguments[:body]
+		        perform_request(method, path, params, body, headers).body
+	        end
+	
+	      end
+	    end
+    end
+  end
+end

data/lib/opensearch/api/namespace/machine_learning/model.rb

@@ -0,0 +1,27 @@
+Dir[File.expand_path('../../actions/**/*.rb', __dir__)].sort.each   { |f| require f }
+
+module OpenSearch
+  module API
+    module MachineLearning
+      module Models
+          module Actions; end
+    
+          # Client for the "machine_learning/models" namespace (includes the {MachineLearning::Models::Actions} methods)
+          #
+          class MachineLearningClient
+            include MachineLearning::Models::Actions
+            include OpenSearch::API::Common::Client::Base
+            include OpenSearch::API::Common::Client
+          end
+    
+          # Proxy method for {MachineLearningModel}, available in the receiving object
+          #
+          def machine_learning
+            @machine_learning ||= MachineLearningClient.new(self)
+          end
+    
+          alias ml machine_learning
+        end
+    end
+  end
+end

data/lib/stretchy/attributes/type/date_time.rb

@@ -24,6 +24,17 @@ module Stretchy::Attributes::Type
   class DateTime < Stretchy::Attributes::Type::Base
     OPTIONS = [:doc_values, :format, :locale, :ignore_malformed, :index, :null_value, :on_script_error, :script, :store, :meta]
     attr_reader *OPTIONS
+    include ActiveModel::Type::Helpers::Timezone
+    include ActiveModel::Type::Helpers::AcceptsMultiparameterTime.new(
+      defaults: { 4 => 0, 5 => 0 }
+    )
+    include ActiveModel::Type::Helpers::TimeValue
+
+    def initialize(**args)
+      @model_format = args.delete(:model_format)
+      super
+    end
+
     def type
       :datetime
     end
@@ -31,5 +42,44 @@ module Stretchy::Attributes::Type
     def type_for_database
       :date
     end
+
+    private
+    def cast_value(value)
+      return apply_seconds_precision(value) unless value.is_a?(::String)
+      return if value.empty?
+
+      fast_string_to_time(value) || fallback_string_to_time(value) || custom_string_to_time(value)
+    end
+
+    # '0.123456' -> 123456
+    # '1.123456' -> 123456
+    def microseconds(time)
+      time[:sec_fraction] ? (time[:sec_fraction] * 1_000_000).to_i : 0
+    end
+
+    def custom_string_to_time(string)
+      ::Date.strptime(string, @model_format)
+    end
+
+    def fallback_string_to_time(string)
+      time_hash = begin
+        ::Date._parse(string)
+      rescue ArgumentError => e
+      end
+      return unless time_hash
+
+      time_hash[:sec_fraction] = microseconds(time_hash)
+
+      new_time(*time_hash.values_at(:year, :mon, :mday, :hour, :min, :sec, :sec_fraction, :offset))
+    end
+
+    def value_from_multiparameter_assignment(values_hash)
+      missing_parameters = [1, 2, 3].delete_if { |key| values_hash.key?(key) }
+      unless missing_parameters.empty?
+        raise ArgumentError, "Provided hash #{values_hash} doesn't contain necessary keys: #{missing_parameters}"
+      end
+      super
+    end
+
   end
 end

data/lib/stretchy/attributes/type/rank_features.rb

@@ -14,11 +14,21 @@ module Stretchy::Attributes::Type
   #   end
   #
   # Returns nothing.
-  class RankFeatures < Stretchy::Attributes::Type::Base
+  class RankFeatures < Stretchy::Attributes::Type::Hash
     OPTIONS = [:positive_score_impact]
 
+    def mappings(name)
+      options = {type: type}
+      OPTIONS.each { |option| options[option] = send(option) unless send(option).nil? }
+      { name =>  options }.as_json 
+    end
+
     def type
       :rank_features
     end
+
+    def type_for_database
+      :rank_features
+    end
   end
 end

data/lib/stretchy/attributes.rb

@@ -43,6 +43,7 @@ module Stretchy
       ActiveModel::Type.register(:percolator, Stretchy::Attributes::Type::Percolator)
       ActiveModel::Type.register(:point, Stretchy::Attributes::Type::Point)
       ActiveModel::Type.register(:rank_feature, Stretchy::Attributes::Type::RankFeature)
+      ActiveModel::Type.register(:rank_features, Stretchy::Attributes::Type::RankFeatures)
 
       ActiveModel::Type.register(:text, Stretchy::Attributes::Type::Text)
       ActiveModel::Type.register(:token_count, Stretchy::Attributes::Type::TokenCount)

data/lib/stretchy/common.rb

@@ -20,6 +20,11 @@ module Stretchy
                 @default_size
             end
 
+            def default_pipeline(pipeline = nil)
+                @default_pipeline = pipeline.to_s unless pipeline.nil?
+                @default_pipeline
+            end
+
             private
 
             # Return a Relation instance to chain queries

data/lib/stretchy/delegation/gateway_delegation.rb

@@ -30,14 +30,20 @@ module Stretchy
           end
       end
 
+      def index_settings(settings={})
+        @index_settings ||= settings
+        @index_settings.merge!(default_pipeline: default_pipeline.to_s) if default_pipeline
+        @index_settings.with_indifferent_access
+      end
+
       def reload_gateway_configuration!
         @gateway = nil
       end
 
       def gateway(&block)
           reload_gateway_configuration! if @gateway && @gateway.client != Stretchy.configuration.client
-            
-          @gateway ||= Stretchy::Repository.create(client: Stretchy.configuration.client, index_name: index_name, klass: base_class, mapping: base_class.attribute_mappings.merge(dynamic: true)) 
+
+          @gateway ||= Stretchy::Repository.create(client: Stretchy.configuration.client, index_name: index_name, klass: base_class, mapping: base_class.attribute_mappings.merge(dynamic: true), settings: index_settings) 
           # block.arity < 1 ? @gateway.instance_eval(&block) : block.call(@gateway) if block_given?
           @gateway
       end

data/lib/stretchy/machine_learning/model.rb

@@ -0,0 +1,192 @@
+module Stretchy 
+  module MachineLearning
+    class Model
+      PRETRAINED_MODELS = {
+        :neural_sparse => {
+          :encoding => 'amazon/neural-sparse/opensearch-neural-sparse-encoding-v1',
+          :encoding_doc => 'amazon/neural-sparse/opensearch-neural-sparse-encoding-doc-v1',
+          :tokenizer => 'amazon/neural-sparse/opensearch-neural-sparse-tokenizer-v1'
+        },
+        :cross_encoder => {
+          :minilm_6 => 'huggingface/cross-encoders/ms-marco-MiniLM-L-6-v2',
+          :minilm_12 => 'huggingface/cross-encoders/ms-marco-MiniLM-L-12-v2'
+        },
+        :sentence_transformers => {
+          :roberta_all => 'huggingface/sentence-transformers/all-distilroberta-v1',
+          :msmarco => 'huggingface/sentence-transformers/msmarco-distilroberta-base-v2',
+          :minilm_6 => 'huggingface/sentence-transformers/all-MiniLM-L6-v2',
+          :minilm_12 => 'huggingface/sentence-transformers/all-MiniLM-L12-v2',
+          :mpnet => 'huggingface/sentence-transformers/all-mpnet-base-v',
+          :multi_qa_minilm_6 => 'huggingface/sentence-transformers/multi-qa-MiniLM-L6-cos-v1',
+          :multi_qa_mpnet => 'huggingface/sentence-transformers/multi-qa-mpnet-base-dot-v1',
+          :paraphrase_minilm_3 => 'huggingface/sentence-transformers/paraphrase-MiniLM-L3-v2',
+          :paraphrase_multilingual_minilm_12 => 'huggingface/sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2',
+          :paraphrase_mpnet => 'huggingface/sentence-transformers/paraphrase-mpnet-base-v2',
+          :multilingual_distiluse_cased => 'huggingface/sentence-transformers/distiluse-base-multilingual-cased-v1'
+        }
+      }
+
+      cattr_reader :client do 
+        Stretchy.configuration.client.ml
+      end
+
+      class << self
+        attr_accessor :model, :group_id 
+
+        def all
+          client.get_model
+        end
+
+        def ml_on_all_nodes!
+          settings = {
+            "persistent": {
+              "plugins": {
+                "ml_commons": {
+                  "only_run_on_ml_node": "false",
+                  "model_access_control_enabled": "true",
+                  "native_memory_threshold": "99"
+                }
+              }
+            }
+          }
+          Stretchy.configuration.client.cluster.put_settings body: settings
+        end
+
+        def ml_on_ml_nodes!
+          settings = {
+            "persistent": {
+              "plugins": {
+                "ml_commons": {
+                  "only_run_on_ml_node": "true",
+                  "model_access_control_enabled": "true",
+                  "native_memory_threshold": "99"
+                }
+              }
+            }
+          }
+          Stretchy.configuration.client.cluster.put_settings body: settings
+        end
+
+        def model_lookup(model)
+          @flattened_models ||= PRETRAINED_MODELS.flat_map do |key, value|
+            value.map do |sub_key, sub_value|
+              ["#{key}_#{sub_key}".to_sym, sub_value]
+            end
+          end.to_h
+
+          @flattened_models[model.to_sym] || model.to_s
+        end
+      end
+
+      attr_accessor :model,
+                    :group_id, 
+                    :version, 
+                    :description, 
+                    :model_format, 
+                    :enabled, 
+                    :connector_id, 
+                    :connector, 
+                    :function_name, 
+                    :model_config, 
+                    :model_content_hash_value, 
+                    :url
+
+      attr_reader :task_id, :model_id, :deploy_id
+
+      def initialize(args = {})
+        model_name = args.delete(:model)
+        args.each do |k,v|
+          self.send("#{k}=", v)
+        end
+        @model = self.class.model_lookup model_name
+      end
+
+      def register
+        begin
+          response = client.register(body: self.to_hash, deploy: true)
+
+          @task_id = response['task_id']
+
+          yield self if block_given? 
+
+          @model_id
+        rescue => e
+          Stretchy.logger.error "Error registering model: #{e.message}"
+          false
+        end
+        true
+      end
+
+      def registered?
+        response = status
+        @model_id = response['model_id'] if response['model_id']
+        response['state'] == 'COMPLETED' && @model_id.present?
+      end
+
+      def status
+        client.get_status(task_id: self.task_id)
+      end
+
+      def deploy
+        @deployed = nil
+        @deploy_id = client.deploy(id: self.model_id)['task_id']
+        yield self if block_given? 
+        @deploy_id
+      end
+
+      def undeploy
+        @deployed = nil
+        response = client.undeploy(id: self.model_id)
+        yield self if block_given? 
+        response
+      end
+
+      def deployed?
+        return @deployed if @deployed
+        response = client.get_model(id: self.model_id)
+        # raise "Model not deployed" if response['model_state'] == 'FAILED'
+        @deployed = response['model_state'] == 'DEPLOYED'
+      end
+
+      def delete
+        client.delete_model(id: self.model_id)
+      end
+
+      def client
+        @@client
+      end
+
+      def find 
+        client.get_model(id: self.model_id)
+      end
+
+      def to_hash
+        {
+          name: self.model,
+          model_group_id: self.group_id,
+          version: self.version,
+          description: self.description,
+          model_format: self.model_format,
+          is_enabled: self.enabled?
+        }.compact
+      end
+
+      def enabled?
+        self.enabled
+      end
+
+      def wait_until_complete(max_attempts: 20, sleep_time: 4)
+        attempts = 0
+        loop do
+          result = yield
+          break if result
+          attempts += 1
+          break if attempts >= max_attempts
+          sleep(sleep_time)
+        end
+      end
+
+
+    end
+  end
+end

data/lib/stretchy/open_search_compatibility.rb

@@ -1,3 +1,5 @@
+require 'opensearch/api/namespace/machine_learning/model'
+
 module Stretchy
     module OpenSearchCompatibility
     extend ActiveSupport::Concern
@@ -79,6 +81,8 @@ module Stretchy
 
             ::Elasticsearch::Persistence::Repository.send(:include, patch)
             ::Elasticsearch::Persistence::Repository.send(:include, store)
+            OpenSearch::API.send(:include, OpenSearch::API::MachineLearning::Models)
+
         end
 
 

data/lib/stretchy/pipeline.rb

@@ -0,0 +1,123 @@
+module Stretchy
+  class Pipeline
+    cattr_reader :client do 
+      Stretchy.configuration.client.ingest 
+    end
+
+    class << self
+      attr_accessor :description, :pipeline_name, :processors
+
+      def pipeline_name(name = nil)
+        return @pipeline_name if name.nil? && @pipeline_name
+        @pipeline_name = name || self.name.split('::').last.underscore
+      end
+
+      def description(desc = nil)
+        @description = desc if desc
+        @description
+      end
+
+      def processor(type, opts = {})
+        @processors ||= []
+        @processors << Stretchy::Pipelines::Processor.new(type, opts)
+      end
+
+      def all
+        begin
+          client.get_pipeline
+        rescue not_found => e
+          return {}
+        end
+      end
+    
+      def find(id)
+        client.get_pipeline(id: id)
+      end
+
+      def simulate(docs, verbose: true)
+        client.simulate(id: self.pipeline_name, body: {docs: docs}, verbose: verbose)
+      end
+
+      # PUT _ingest/pipeline/<pipeline-name>
+      def create!
+        client.put_pipeline(id: self.pipeline_name, body: self.to_hash)
+      end
+
+      # DELETE _ingest/pipeline/<pipeline-name>
+      def delete!
+        client.delete_pipeline(id: self.pipeline_name)
+      end
+
+      def exists?
+        begin
+          self.find(self.pipeline_name).present?
+        rescue not_found => e
+          return false
+        end
+      end
+
+
+      def to_hash
+        {
+          description: self.description,
+          processors: self.processors.map(&:to_hash)
+        }.as_json
+      end
+
+      protected 
+      def not_found
+        @not_found ||= Object.const_get("#{client.class.name.split('::').first}::Transport::Transport::Errors::NotFound") 
+      end
+
+    end
+
+    attr_accessor :description, :pipeline_name, :processors
+
+    def initialize
+      @description = self.class.description
+      @pipeline_name = self.class.pipeline_name
+      @processors = self.class.processors
+    end
+
+    # GET _ingest/pipeline/<pipeline-name>
+    def find
+      self.class.find(self.pipeline_name)
+    end
+
+    # Simulates the pipeline.
+    #
+    # Request body fields
+    #
+    # The following table lists the request body fields used to run a pipeline.
+    #
+    # Field	Required	Type	Description
+    # docs	Required	Array	The documents to be used to test the pipeline.
+    # pipeline	Optional	Object	The pipeline to be simulated. If the pipeline identifier is not included, then the response simulates the latest pipeline created.
+    # The docs field can include subfields listed in the following table.
+    #
+    # Field	Required	Type	Description
+    # source	Required	Object	The document’s JSON body.
+    # id	Optional	String	A unique document identifier. The identifier cannot be used elsewhere in the index.
+    # index	Optional	String	The index where the document’s transformed data appears.
+    def simulate(docs, verbose: true)
+      self.class.simulate(docs, verbose: verbose)
+    end
+    
+    def exists?
+      self.class.exists?
+    end
+
+    def to_hash
+      {
+        description: self.description,
+        processors: self.processors.map(&:to_hash)
+      }.as_json
+    end
+
+    def client
+      @client ||= Stretchy.configuration.client.ingest
+    end
+
+
+  end
+end

data/lib/stretchy/pipelines/processor.rb

@@ -0,0 +1,55 @@
+module Stretchy::Pipelines
+  # Creates a new processor for a pipeline
+  # 
+  # Processor type	      Description
+  # append	              Adds one or more values to a field in a document.
+  # bytes	                Converts a human-readable byte value to its value in bytes.
+  # convert	              Changes the data type of a field in a document.
+  # copy	                Copies an entire object in an existing field to another field.
+  # csv	                  Extracts CSVs and stores them as individual fields in a document.
+  # date	                Parses dates from fields and then uses the date or timestamp as the timestamp for a document.
+  # date_index_name	      Indexes documents into time-based indexes based on a date or timestamp field in a document.
+  # dissect	              Extracts structured fields from a text field using a defined pattern.
+  # dot_expander	        Expands a field with dots into an object field.
+  # drop	                Drops a document without indexing it or raising any errors.
+  # fail    	            Raises an exception and stops the execution of a pipeline.
+  # foreach	              Allows for another processor to be applied to each element of an array or an object field in a document.
+  # geoip	                Adds information about the geographical location of an IP address.
+  # geojson-feature	      Indexes GeoJSON data into a geospatial field.
+  # grok	                Parses and structures unstructured data using pattern matching.
+  # gsub	                Replaces or deletes substrings within a string field of a document.
+  # html_strip	          Removes HTML tags from a text field and returns the plain text content.
+  # ip2geo	              Adds information about the geographical location of an IPv4 or IPv6 address.
+  # join	                Concatenates each element of an array into a single string using a separator character between each element.
+  # json	                Converts a JSON string into a structured JSON object.
+  # kv	                  Automatically parses key-value pairs in a field.
+  # lowercase	            Converts text in a specific field to lowercase letters.
+  # pipeline	            Runs an inner pipeline.
+  # remove	              Removes fields from a document.
+  # script	              Runs an inline or stored script on incoming documents.
+  # set	                  Sets the value of a field to a specified value.
+  # sort	                Sorts the elements of an array in ascending or descending order.
+  # sparse_encoding	      Generates a sparse vector/token and weights from text fields for neural sparse search using sparse retrieval.
+  # split	                Splits a field into an array using a separator character.
+  # text_embedding	      Generates vector embeddings from text fields for semantic search.
+  # text_image_embedding	Generates combined vector embeddings from text and image fields for multimodal neural search.
+  # trim	                Removes leading and trailing white space from a string field.
+  # uppercase	            Converts text in a specific field to uppercase letters.
+  # urldecode	            Decodes a string from URL-encoded format.
+  # user_agent	          Extracts details from the user agent sent by a browser to its web requests.
+  #
+  class Processor
+    
+    attr_reader :type, :opts, :description
+
+    def initialize(type, opts = {})
+      @type = type
+      @description = opts[:description]
+      @opts = opts
+    end
+
+    def to_hash
+      { type => @opts }
+    end
+  end
+end

data/lib/stretchy/querying.rb

@@ -3,6 +3,7 @@ module Stretchy
     delegate :first, :first!, :last, :last!, :exists?, :has_field, :any?, :many?, to: :all
     delegate :order, :limit, :size, :sort, :rewhere, :eager_load, :includes,  :create_with, :none, :unscope, to: :all
     delegate :or_filter, :fields, :source, :highlight, to: :all
+    delegate :neural_sparse, :neural, :hybrid, to: :all
     delegate *Stretchy::Relations::AggregationMethods::AGGREGATION_METHODS, to: :all
 
     delegate :skip_callbacks, :routing, :search_options, to: :all

data/lib/stretchy/rails/instrumentation/publishers.rb

@@ -0,0 +1,31 @@
+module Stretchy
+    module Rails
+        module Instrumentation
+            module Publishers
+
+                module Record 
+
+                    extend ActiveSupport::Concern
+
+                    included do
+                        unless method_defined?(:search_without_instrumentation!)
+                            alias_method :search_without_instrumentation!, :search
+                            alias_method :search, :search_with_instrumentation!
+                        end
+                    end
+
+                    def search_with_instrumentation!(query_or_definition, options={})
+                        ActiveSupport::Notifications.instrument "search.stretchy", 
+                        name: "Search", 
+                        klass: self.base_class.to_s,
+                        search: {index: self.index_name, body: query_or_definition }.merge(options) do
+                            search_without_instrumentation!(query_or_definition, options)
+                        end
+                    end
+
+                end
+
+            end
+        end
+    end
+end