stretchy-model 0.6.5 → 0.7.0

data/lib/stretchy/pipelines/processor.rb

@@ -1,43 +1,45 @@
 module Stretchy::Pipelines
   # Creates a new processor for a pipeline
   # 
-  # Processor type	      Description
-  # append	              Adds one or more values to a field in a document.
-  # bytes	                Converts a human-readable byte value to its value in bytes.
-  # convert	              Changes the data type of a field in a document.
-  # copy	                Copies an entire object in an existing field to another field.
-  # csv	                  Extracts CSVs and stores them as individual fields in a document.
-  # date	                Parses dates from fields and then uses the date or timestamp as the timestamp for a document.
-  # date_index_name	      Indexes documents into time-based indexes based on a date or timestamp field in a document.
-  # dissect	              Extracts structured fields from a text field using a defined pattern.
-  # dot_expander	        Expands a field with dots into an object field.
-  # drop	                Drops a document without indexing it or raising any errors.
-  # fail    	            Raises an exception and stops the execution of a pipeline.
-  # foreach	              Allows for another processor to be applied to each element of an array or an object field in a document.
-  # geoip	                Adds information about the geographical location of an IP address.
-  # geojson-feature	      Indexes GeoJSON data into a geospatial field.
-  # grok	                Parses and structures unstructured data using pattern matching.
-  # gsub	                Replaces or deletes substrings within a string field of a document.
-  # html_strip	          Removes HTML tags from a text field and returns the plain text content.
-  # ip2geo	              Adds information about the geographical location of an IPv4 or IPv6 address.
-  # join	                Concatenates each element of an array into a single string using a separator character between each element.
-  # json	                Converts a JSON string into a structured JSON object.
-  # kv	                  Automatically parses key-value pairs in a field.
-  # lowercase	            Converts text in a specific field to lowercase letters.
-  # pipeline	            Runs an inner pipeline.
-  # remove	              Removes fields from a document.
-  # script	              Runs an inline or stored script on incoming documents.
-  # set	                  Sets the value of a field to a specified value.
-  # sort	                Sorts the elements of an array in ascending or descending order.
-  # sparse_encoding	      Generates a sparse vector/token and weights from text fields for neural sparse search using sparse retrieval.
-  # split	                Splits a field into an array using a separator character.
-  # text_embedding	      Generates vector embeddings from text fields for semantic search.
-  # text_image_embedding	Generates combined vector embeddings from text and image fields for multimodal neural search.
-  # trim	                Removes leading and trailing white space from a string field.
-  # uppercase	            Converts text in a specific field to uppercase letters.
-  # urldecode	            Decodes a string from URL-encoded format.
-  # user_agent	          Extracts details from the user agent sent by a browser to its web requests.
+  # Below is a list of processors that can be used in a pipeline. Each processor has a specific function that can be applied to a document.
+  # 
+  # For more information on processors, see the [Elasticsearch](https://www.elastic.co/guide/en/elasticsearch/reference/current/ingest-processors.html).
+  # or [OpenSearch](https://opensearch.org/docs/latest/ingest-pipelines/processors/index-processors/) documentation.
   #
+  # | type | Description |
+  # | --- | --- |
+  # | append | Adds one or more values to a field in a document. |
+  # | bytes | Converts a human-readable byte value to its value in bytes. |
+  # | convert | Changes the data type of a field in a document. |
+  # | copy | Copies an entire object in an existing field to another field. |
+  # | csv | Extracts CSVs and stores them as individual fields in a document. |
+  # | date | Parses dates from fields and then uses the date or timestamp as the timestamp for a document. |
+  # | date_index_name | Indexes documents into time-based indexes based on a date or timestamp field in a document. |
+  # | dissect | Extracts structured fields from a text field using a defined pattern. |
+  # | dot_expander | Expands a field with dots into an object field. |
+  # | drop | Drops a document without indexing it or raising any errors. |
+  # | fail | Raises an exception and stops the execution of a pipeline. |
+  # | foreach | Allows for another processor to be applied to each element of an array or an object field in a document. |
+  # | geoip | Adds information about the geographical location of an IP address. |
+  # | geojson-feature | Indexes GeoJSON data into a geospatial field. |
+  # | grok | Parses and structures unstructured data using pattern matching. |
+  # | gsub | Replaces or deletes substrings within a string field of a document. |
+  # | html_strip | Removes HTML tags from a text field and returns the plain text content. |
+  # | ip2geo | Adds information about the geographical location of an IPv4 or IPv6 address. |
+  # | join | Concatenates each element of an array into a single string using a separator character between each element. |
+  # | json | Converts a JSON string into a structured JSON object. |
+  # | kv | Automatically parses key-value pairs in a field. |
+  # | lowercase | Converts text in a specific field to lowercase letters. |
+  # | pipeline | Runs an inner pipeline. |
+  # | remove | Removes fields from a document. |
+  # | script | Runs an inline or stored script on incoming documents. |
+  # | set | Sets the value of a field to a specified value. |
+  # | sort | Sorts the elements of an array in ascending or descending order. |
+  # | sparse_encoding | Generates a sparse vector/token and weights from text fields for neural sparse search using sparse retrieval. |
+  # | split | Splits a field into an array using a separator character. |
+  # | text_embedding | Generates vector embeddings from text fields for semantic search. |
+  # | text_image_embedding | Generates combined vector embeddings from text and image fields for multimodal neural search. |
+
   class Processor
     
     attr_reader :type, :opts, :description
@@ -46,9 +48,11 @@ module Stretchy::Pipelines
       @type = type
       @description = opts[:description]
       @opts = opts
+      @model = opts.delete(:model)
     end
 
     def to_hash
+      @opts[:model_id] = @model.model_id if @model
       { type => @opts }
     end
   end

data/lib/stretchy/querying.rb

@@ -1,13 +1,13 @@
 module Stretchy
   module Querying
-    delegate :first, :first!, :last, :last!, :exists?, :has_field, :any?, :many?, to: :all
-    delegate :order, :limit, :size, :sort, :rewhere, :eager_load, :includes,  :create_with, :none, :unscope, to: :all
-    delegate :or_filter, :fields, :source, :highlight, to: :all
-    delegate :neural_sparse, :neural, :hybrid, to: :all
-    delegate *Stretchy::Relations::AggregationMethods::AGGREGATION_METHODS, to: :all
+    delegate :exists?, :any?, :many?, :includes, to: :all
+    delegate :rewhere, :eager_load, :create_with, :unscoped, to: :all
+
+    delegate *Stretchy::Relations::FinderMethods::METHODS, to: :all
+    delegate *Stretchy::Relations::SearchOptionMethods::METHODS, to: :all
+    delegate *Stretchy::Relations::QueryMethods.registry, to: :all
+    delegate *Stretchy::Relations::AggregationMethods.registry, to: :all
 
-    delegate :skip_callbacks, :routing, :search_options, to: :all
-    delegate :must, :must_not, :should, :where_not, :where, :filter_query, :query_string, :regexp, to: :all
 
     def fetch_results(es)
       if es.count?
@@ -17,6 +17,5 @@ module Stretchy
       end
     end
 
-
   end
 end

data/lib/stretchy/rails/railtie.rb

@@ -0,0 +1,11 @@
+module Stretchy
+  module Rails
+    class Railtie < ::Rails::Railtie
+
+      rake_tasks do
+        load "#{__dir__}/tasks/stretchy.rake"
+      end
+
+    end
+  end
+end

data/lib/stretchy/rails/tasks/connector/create.rake

@@ -0,0 +1,32 @@
+namespace :stretchy do
+  namespace :connector do
+    desc "Create connector"
+    task create: :environment do
+      klass = ENV['MODEL']
+      Rails.application.eager_load!
+      models = klass.nil? ? Stretchy::MachineLearning::Connector.descendants : [klass.constantize]
+        models.each do |model|
+          spinner = TTY::Spinner.new("Creating Connector #{model} ".ljust(JUSTIFICATION) + ":spinner", format: :dots)
+          spinner.auto_spin
+
+          # if model.exists?
+          #   spinner.stop(Rainbow("[EXISTS]").yellow)
+          #   next
+          # end
+
+          begin
+            response = model.create!
+          rescue Stretchy::MachineLearning::Errors::ConnectorMissingError => e
+            spinner.stop(Rainbow("[FAILED]").red)
+            next
+          end
+          status = if response
+            Rainbow("[SUCCESS]").green
+          else
+            Rainbow("[FAILED]").red
+          end
+          spinner.stop(status)
+        end
+      end
+  end
+end

data/lib/stretchy/rails/tasks/connector/delete.rake

@@ -0,0 +1,27 @@
+namespace :stretchy do
+  namespace :connector do
+    desc "Delete Connector"
+    task delete: :environment do
+      klass = ENV['MODEL']
+      Rails.application.eager_load!
+      models = klass.nil? ? Stretchy::MachineLearning::Connector.descendants : [klass.constantize]
+        models.each do |model|
+          spinner = TTY::Spinner.new("Deleting Connector #{model} ".ljust(JUSTIFICATION) + ":spinner", format: :dots)
+          spinner.auto_spin
+
+          begin
+            response = model.delete!
+          rescue Stretchy::MachineLearning::Errors::ConnectorMissingError => e
+            spinner.stop(Rainbow("[MISSING]").white)
+            next
+          end
+          status = if response['result'] == 'deleted'
+            Rainbow("[SUCCESS]").green
+          else
+            Rainbow("[#{response['result'].upcase}]").white
+          end
+          spinner.stop(status)
+        end
+      end
+  end
+end

data/lib/stretchy/rails/tasks/connector/status.rake

@@ -0,0 +1,31 @@
+namespace :stretchy do
+  namespace :connector do
+
+    desc "Check the status of all connectors"
+    task status: :environment do
+      klass = ENV['MODEL']
+
+      Rails.application.eager_load!
+      models = klass.nil? ? Stretchy::MachineLearning::Connector.descendants : [klass.constantize]
+
+      puts Rainbow("Connector").color :gray
+      models.each do |model|
+        
+        begin
+        response = model.find
+        rescue Stretchy::MachineLearning::Errors::ConnectorMissingError => e
+          puts "#{model}".ljust(JUSTIFICATION) + Rainbow("[MISSING]").white
+          next
+        end
+        status = if response
+          Rainbow("[CREATED]").green.bright
+        else
+          Rainbow("[MISSING]").white
+        end
+        puts "#{model}".ljust(JUSTIFICATION) + status
+      end
+      puts "\n\n"
+    end
+
+  end
+end

data/lib/stretchy/rails/tasks/connector/update.rake

@@ -0,0 +1,32 @@
+namespace :stretchy do
+  namespace :connector do
+    desc "Update connector"
+    task update: :environment do
+      klass = ENV['MODEL']
+      Rails.application.eager_load!
+      models = klass.nil? ? Stretchy::MachineLearning::Connector.descendants : [klass.constantize]
+        models.each do |model|
+          spinner = TTY::Spinner.new("Updating Connector #{model} ".ljust(JUSTIFICATION) + ":spinner", format: :dots)
+          spinner.auto_spin
+
+          # if model.exists?
+          #   spinner.stop(Rainbow("[EXISTS]").yellow)
+          #   next
+          # end
+
+          begin
+            response = model.update!
+          rescue Stretchy::MachineLearning::Errors::ConnectorMissingError => e
+            spinner.stop(Rainbow("[FAILED]").red)
+            next
+          end
+          status = if response
+            Rainbow("[SUCCESS]").green
+          else
+            Rainbow("[FAILED]").red
+          end
+          spinner.stop(status)
+        end
+      end
+  end
+end

data/lib/stretchy/rails/tasks/index/create.rake

@@ -0,0 +1,28 @@
+namespace :stretchy do
+  namespace :index do
+    desc "Create indices"
+    task create: :environment do
+      klass = ENV['MODEL']
+      Rails.application.eager_load!
+      models = klass.nil? ? StretchyModel.descendants : [klass.constantize]
+        models.each do |model|
+          next if model.name == "Stretchy::MachineLearning::Registry"
+          spinner = TTY::Spinner.new("Creating index #{model} (#{model.index_name}) ".ljust(JUSTIFICATION) + ":spinner", format: :dots)
+          spinner.auto_spin
+          if model.index_exists?
+            spinner.stop( Rainbow("[EXISTS]").orange)
+            next
+          end
+
+          response = model.create_index!
+          status = if response
+            Rainbow("[SUCCESS]").green
+          else
+            Rainbow("[FAILED]").red
+          end
+
+          spinner.stop(status)
+        end
+      end
+  end
+end

data/lib/stretchy/rails/tasks/index/delete.rake

@@ -0,0 +1,27 @@
+namespace :stretchy do
+  namespace :index do
+    desc "Delete indices"
+    task delete: :environment do
+      klass = ENV['MODEL']
+      models = klass.nil? ? StretchyModel.descendants : [klass.constantize]
+        models.each do |model|
+          next if model.name == "Stretchy::MachineLearning::Registry"
+          spinner = TTY::Spinner.new("Deleting index #{model} (#{model.index_name}) ".ljust(JUSTIFICATION) + ":spinner", format: :dots)
+          spinner.auto_spin
+          unless model.index_exists?
+            spinner.stop( Rainbow("[MISSING]").white)
+            next
+          end
+
+          response = model.delete_index!
+          status = if response
+            Rainbow("[SUCCESS]").green
+          else
+            Rainbow("[FAILED]").red
+          end
+
+          spinner.stop(status)
+        end
+      end
+  end
+end

data/lib/stretchy/rails/tasks/index/status.rake

@@ -0,0 +1,23 @@
+namespace :stretchy do
+  namespace :index do
+    desc "Check the status of all indexes"
+    task status: :environment do
+      klass = ENV['MODEL']
+
+      Rails.application.eager_load!
+      models = klass.nil? ? StretchyModel.descendants : [klass.constantize]
+
+      puts Rainbow("Indexes").color :gray
+      models.each do |model|
+        response = model.index_exists?
+        status = if response
+          Rainbow("[CREATED]").green.bright
+        else
+          Rainbow("[MISSING]").white
+        end
+        puts "#{model.index_name}".ljust(JUSTIFICATION) + status
+      end
+      puts "\n\n"
+    end
+  end
+end

data/lib/stretchy/rails/tasks/ml/delete.rake

@@ -0,0 +1,25 @@
+namespace :stretchy do
+  namespace :ml do
+
+    desc "Delete the model ENV['MODEL']"
+    task delete: :environment do
+      klass = ENV['MODEL']
+      Rails.application.eager_load!
+      models = klass.nil? ? Stretchy::MachineLearning::Model.descendants : [klass.constantize]
+
+      models.each do |model|
+        spinner = TTY::Spinner.new(("Deleting #{model} ".ljust(JUSTIFICATION) + ":spinner"), format: :dots)
+        spinner.auto_spin
+        unless model.registered?
+          spinner.stop(Rainbow("[MISSING]").white)
+          next
+        end
+
+        model.delete
+
+        status = !model.registered? ? Rainbow("[DELETED]").green : Rainbow("[NOT DELETED]").yellow
+        spinner.stop(status)
+      end
+    end
+  end
+end

data/lib/stretchy/rails/tasks/ml/deploy.rake

@@ -0,0 +1,78 @@
+namespace :stretchy do
+  namespace :ml do
+
+    desc "Register the model ENV['MODEL']"
+    task register: :environment do
+      klass = ENV['MODEL']
+      Rails.application.eager_load!
+      models = klass.nil? ? Stretchy::MachineLearning::Model.descendants : [klass.constantize]
+
+      models.each do |model|
+        spinner = TTY::Spinner.new(("Registering #{model} ".ljust(JUSTIFICATION) + ":spinner"), format: :dots)
+        spinner.auto_spin
+        if model.registered?
+          spinner.stop(Rainbow("[SUCCESS]").green)
+          next
+        end
+
+        model.register do  |m|
+          m.wait_until_complete do
+            m.registered?
+          end
+        end
+
+        status = model.registered? ? Rainbow("[SUCCESS]").green : Rainbow("[FAILED]").red
+        spinner.stop(status)
+      end
+    end
+
+    desc "Undeploy the model ENV['MODEL']"
+    task undeploy: :environment do
+      klass = ENV['MODEL']
+      Rails.application.eager_load!
+      models = klass.nil? ? Stretchy::MachineLearning::Model.descendants : [klass.constantize]
+      models.each do |model|
+        spinner = TTY::Spinner.new("Undeploying #{model} ". ljust(JUSTIFICATION) + ":spinner", format: :dots)
+        spinner.auto_spin
+
+        unless model.deployed?
+          spinner.stop(Rainbow("[NOT DEPLOYED]").white)
+          next
+        end
+
+        model.undeploy do |m|
+          m.wait_until_complete do
+            !m.deployed?
+          end
+        end
+        status = !model.deployed? ?  Rainbow("[SUCCESS]").green : Rainbow("[DEPLOYED]").yellow
+        spinner.stop(status)
+      end
+    end
+
+    desc "Deploy the model ENV['MODEL']"
+    task deploy: :environment do
+      klass = ENV['MODEL']
+      Rails.application.eager_load!
+      models = klass.nil? ? Stretchy::MachineLearning::Model.descendants : [klass.constantize]
+      models.each do |model|
+        spinner = TTY::Spinner.new("Deploying #{model} ".ljust(JUSTIFICATION) + ":spinner", format: :dots)
+        spinner.auto_spin
+
+        if model.deployed?
+          spinner.stop(Rainbow("[SUCCESS]").green)
+          next
+        end
+
+        model.deploy do |m|
+          m.wait_until_complete do
+            m.deployed?
+          end
+        end
+        status = model.deployed? ? Rainbow("[SUCCESS]").green : Rainbow("[FAILED]").red 
+        spinner.stop(status)
+      end
+    end
+
+  end
+end

data/lib/stretchy/rails/tasks/ml/status.rake

@@ -0,0 +1,31 @@
+namespace :stretchy do
+  namespace :ml do
+
+    desc "Check the status of all ml models"
+    task status: :environment do
+      klass = ENV['MODEL']
+
+      Rails.application.eager_load!
+      models = klass.nil? ? Stretchy::MachineLearning::Model.descendants : [klass.constantize]
+
+      puts Rainbow("Machine Learning").color :gray
+      models.each do |model|
+        
+        response = model.find['model_state']
+        status = case response
+          when 'DEPLOYED'
+            Rainbow("[#{response}]").green.bright
+          when 'DEPLOY_FAILED'
+            Rainbow("[#{response}]").red.bright
+          when 'CREATED'
+            Rainbow("[#{response}]").yellow.bright
+        else
+          Rainbow("[MISSING]").white
+        end
+        puts "#{model}".ljust(JUSTIFICATION) + status
+      end
+      puts "\n\n"
+    end
+
+  end
+end

data/lib/stretchy/rails/tasks/pipeline/create.rake

@@ -0,0 +1,27 @@
+namespace :stretchy do
+  namespace :pipeline do
+    desc "Create pipeline"
+    task create: :environment do
+      klass = ENV['MODEL']
+      Rails.application.eager_load!
+      models = klass.nil? ? Stretchy::Pipeline.descendants : [klass.constantize]
+        models.each do |model|
+          spinner = TTY::Spinner.new("Creating Pipeline #{model} ".ljust(JUSTIFICATION) + ":spinner", format: :dots)
+          spinner.auto_spin
+
+          begin
+            response = model.create!
+          rescue => e
+            spinner.stop(Rainbow("[FAILED]").red)
+            next
+          end
+          status = if response['acknowledged']
+            Rainbow("[SUCCESS]").green
+          else
+            Rainbow("[FAILED]").red
+          end
+          spinner.stop(status)
+        end
+      end
+  end
+end

data/lib/stretchy/rails/tasks/pipeline/delete.rake

@@ -0,0 +1,26 @@
+namespace :stretchy do
+  namespace :pipeline do
+    desc "Delete pipeline"
+    task delete: :environment do
+      klass = ENV['MODEL']
+      models = klass.nil? ? Stretchy::Pipeline.descendants : [klass.constantize]
+        models.each do |model|
+          spinner = TTY::Spinner.new("Deleting Pipeline #{model} ".ljust(JUSTIFICATION) + ":spinner", format: :dots)
+          spinner.auto_spin
+
+          unless model.exists?
+            spinner.stop(Rainbow("[MISSING]").white)
+            next
+          end
+
+          response = model.delete!
+          status = if response['acknowledged']
+            Rainbow("[SUCCESS]").green
+          else
+            Rainbow("[FAILED]").red
+          end
+          spinner.stop(status)
+        end
+      end
+  end
+end

data/lib/stretchy/rails/tasks/pipeline/status.rake

@@ -0,0 +1,25 @@
+namespace :stretchy do
+  namespace :pipeline do
+
+    desc "Check the status of all pipelines"
+    task status: :environment do
+      klass = ENV['MODEL']
+
+      Rails.application.eager_load!
+      models = klass.nil? ? Stretchy::Pipeline.descendants : [klass.constantize]
+
+      puts Rainbow("Pipelines").color :gray
+      models.each do |model|
+        response = model.exists?
+        status = if response
+          Rainbow("[CREATED]").green.bright
+        else
+          Rainbow("[MISSING]").white
+        end
+        puts "#{model.pipeline_name}".ljust(JUSTIFICATION) + status
+      end
+      puts "\n\n"
+    end
+
+  end
+end

data/lib/stretchy/rails/tasks/status.rake

@@ -0,0 +1,15 @@
+namespace :stretchy do
+  namespace :status do
+    desc "Check the status of all indexes"
+    task all: :environment do
+      Rake::Task['stretchy:index:status'].invoke
+      Rake::Task['stretchy:pipeline:status'].invoke
+      Rake::Task['stretchy:ml:status'].invoke
+      Rake::Task['stretchy:connector:status'].invoke
+    end
+  end
+
+  desc "Check the status of all indexes, pipelines and ml models"
+  task status: 'stretchy:status:all' do
+  end
+end

data/lib/stretchy/rails/tasks/stretchy.rake

@@ -0,0 +1,42 @@
+require 'stretchy-model'
+require 'tty-spinner'
+
+JUSTIFICATION = 90 unless defined?(JUSTIFICATION)
+
+path = File.expand_path(__dir__)
+Dir.glob("#{path}/**/*.rake").each do |f|
+  import f unless f == __FILE__
+end
+
+namespace :stretchy do
+  desc "Create all indexes, pipelines and deploy all models"
+  task up: :environment do
+    Rake::Task['stretchy:connector:create'].invoke
+    Rake::Task['stretchy:ml:register'].invoke
+    Rake::Task['stretchy:ml:deploy'].invoke
+    Rake::Task['stretchy:pipeline:create'].invoke
+    Rake::Task['stretchy:index:create'].invoke
+  end
+
+  desc "Delete all indexes, pipelines and undeploy all models"
+  task down: :environment do
+    Rake::Task['stretchy:ml:undeploy'].invoke
+    Rake::Task['stretchy:ml:delete'].invoke
+    Rake::Task['stretchy:connector:delete'].invoke
+    Rake::Task['stretchy:pipeline:delete'].invoke
+    Rake::Task['stretchy:index:delete'].invoke
+  end
+
+  desc "Enable Machine Learning on all nodes"
+  task ml_on_all_nodes: :environment do
+    puts "Enabling Machine Learning on all nodes..."
+    puts Stretchy::MachineLearning::Model.ml_on_all_nodes!
+  end
+
+  desc "Machine Learning on ML nodes only"
+  task ml_on_ml_nodes: :environment do
+    puts "Enabling Machine Learning on ML nodes only..."
+    puts Stretchy::MachineLearning::Model.ml_on_ml_nodes!
+  end
+
+end

data/lib/stretchy/record.rb

@@ -16,12 +16,12 @@ module Stretchy
                 include ActiveModel::Serializers::JSON
 
                 include Stretchy::Model::Callbacks
+                include Stretchy::Model::Common
+                include Stretchy::Model::Persistence
+                include Stretchy::Model::Refreshable
                 include Stretchy::Indexing::Bulk
-                include Stretchy::Persistence
                 include Stretchy::Associations 
-                include Stretchy::Refreshable
-                include Stretchy::Common
-                include Stretchy::Scoping
+                include Stretchy::Relations::Scoping
                 include Stretchy::Utils
                 include Stretchy::SharedScopes
                 include Stretchy::Attributes
@@ -46,6 +46,7 @@ module Stretchy
                 def initialize(attributes = {})
                     @highlights = attributes.delete(:_highlights) 
                     super(attributes)
+                    run_callbacks :initialize 
                 end
 
             end