stretchy-model 0.6.6 → 0.7.0

data/lib/stretchy/rails/tasks/index/status.rake

@@ -0,0 +1,23 @@
+namespace :stretchy do
+  namespace :index do
+    desc "Check the status of all indexes"
+    task status: :environment do
+      klass = ENV['MODEL']
+
+      Rails.application.eager_load!
+      models = klass.nil? ? StretchyModel.descendants : [klass.constantize]
+
+      puts Rainbow("Indexes").color :gray
+      models.each do |model|
+        response = model.index_exists?
+        status = if response
+          Rainbow("[CREATED]").green.bright
+        else
+          Rainbow("[MISSING]").white
+        end
+        puts "#{model.index_name}".ljust(JUSTIFICATION) + status
+      end
+      puts "\n\n"
+    end
+  end
+end

data/lib/stretchy/rails/tasks/ml/delete.rake

@@ -0,0 +1,25 @@
+namespace :stretchy do
+  namespace :ml do
+
+    desc "Delete the model ENV['MODEL']"
+    task delete: :environment do
+      klass = ENV['MODEL']
+      Rails.application.eager_load!
+      models = klass.nil? ? Stretchy::MachineLearning::Model.descendants : [klass.constantize]
+
+      models.each do |model|
+        spinner = TTY::Spinner.new(("Deleting #{model} ".ljust(JUSTIFICATION) + ":spinner"), format: :dots)
+        spinner.auto_spin
+        unless model.registered?
+          spinner.stop(Rainbow("[MISSING]").white)
+          next
+        end
+
+        model.delete
+
+        status = !model.registered? ? Rainbow("[DELETED]").green : Rainbow("[NOT DELETED]").yellow
+        spinner.stop(status)
+      end
+    end
+  end
+end

data/lib/stretchy/rails/tasks/ml/deploy.rake

@@ -0,0 +1,78 @@
+namespace :stretchy do
+  namespace :ml do
+
+    desc "Register the model ENV['MODEL']"
+    task register: :environment do
+      klass = ENV['MODEL']
+      Rails.application.eager_load!
+      models = klass.nil? ? Stretchy::MachineLearning::Model.descendants : [klass.constantize]
+
+      models.each do |model|
+        spinner = TTY::Spinner.new(("Registering #{model} ".ljust(JUSTIFICATION) + ":spinner"), format: :dots)
+        spinner.auto_spin
+        if model.registered?
+          spinner.stop(Rainbow("[SUCCESS]").green)
+          next
+        end
+
+        model.register do  |m|
+          m.wait_until_complete do
+            m.registered?
+          end
+        end
+
+        status = model.registered? ? Rainbow("[SUCCESS]").green : Rainbow("[FAILED]").red
+        spinner.stop(status)
+      end
+    end
+
+    desc "Undeploy the model ENV['MODEL']"
+    task undeploy: :environment do
+      klass = ENV['MODEL']
+      Rails.application.eager_load!
+      models = klass.nil? ? Stretchy::MachineLearning::Model.descendants : [klass.constantize]
+      models.each do |model|
+        spinner = TTY::Spinner.new("Undeploying #{model} ". ljust(JUSTIFICATION) + ":spinner", format: :dots)
+        spinner.auto_spin
+
+        unless model.deployed?
+          spinner.stop(Rainbow("[NOT DEPLOYED]").white)
+          next
+        end
+
+        model.undeploy do |m|
+          m.wait_until_complete do
+            !m.deployed?
+          end
+        end
+        status = !model.deployed? ?  Rainbow("[SUCCESS]").green : Rainbow("[DEPLOYED]").yellow
+        spinner.stop(status)
+      end
+    end
+
+    desc "Deploy the model ENV['MODEL']"
+    task deploy: :environment do
+      klass = ENV['MODEL']
+      Rails.application.eager_load!
+      models = klass.nil? ? Stretchy::MachineLearning::Model.descendants : [klass.constantize]
+      models.each do |model|
+        spinner = TTY::Spinner.new("Deploying #{model} ".ljust(JUSTIFICATION) + ":spinner", format: :dots)
+        spinner.auto_spin
+
+        if model.deployed?
+          spinner.stop(Rainbow("[SUCCESS]").green)
+          next
+        end
+
+        model.deploy do |m|
+          m.wait_until_complete do
+            m.deployed?
+          end
+        end
+        status = model.deployed? ? Rainbow("[SUCCESS]").green : Rainbow("[FAILED]").red 
+        spinner.stop(status)
+      end
+    end
+
+  end
+end

data/lib/stretchy/rails/tasks/ml/status.rake

@@ -0,0 +1,31 @@
+namespace :stretchy do
+  namespace :ml do
+
+    desc "Check the status of all ml models"
+    task status: :environment do
+      klass = ENV['MODEL']
+
+      Rails.application.eager_load!
+      models = klass.nil? ? Stretchy::MachineLearning::Model.descendants : [klass.constantize]
+
+      puts Rainbow("Machine Learning").color :gray
+      models.each do |model|
+        
+        response = model.find['model_state']
+        status = case response
+          when 'DEPLOYED'
+            Rainbow("[#{response}]").green.bright
+          when 'DEPLOY_FAILED'
+            Rainbow("[#{response}]").red.bright
+          when 'CREATED'
+            Rainbow("[#{response}]").yellow.bright
+        else
+          Rainbow("[MISSING]").white
+        end
+        puts "#{model}".ljust(JUSTIFICATION) + status
+      end
+      puts "\n\n"
+    end
+
+  end
+end

data/lib/stretchy/rails/tasks/pipeline/create.rake

@@ -0,0 +1,27 @@
+namespace :stretchy do
+  namespace :pipeline do
+    desc "Create pipeline"
+    task create: :environment do
+      klass = ENV['MODEL']
+      Rails.application.eager_load!
+      models = klass.nil? ? Stretchy::Pipeline.descendants : [klass.constantize]
+        models.each do |model|
+          spinner = TTY::Spinner.new("Creating Pipeline #{model} ".ljust(JUSTIFICATION) + ":spinner", format: :dots)
+          spinner.auto_spin
+
+          begin
+            response = model.create!
+          rescue => e
+            spinner.stop(Rainbow("[FAILED]").red)
+            next
+          end
+          status = if response['acknowledged']
+            Rainbow("[SUCCESS]").green
+          else
+            Rainbow("[FAILED]").red
+          end
+          spinner.stop(status)
+        end
+      end
+  end
+end

data/lib/stretchy/rails/tasks/pipeline/delete.rake

@@ -0,0 +1,26 @@
+namespace :stretchy do
+  namespace :pipeline do
+    desc "Delete pipeline"
+    task delete: :environment do
+      klass = ENV['MODEL']
+      models = klass.nil? ? Stretchy::Pipeline.descendants : [klass.constantize]
+        models.each do |model|
+          spinner = TTY::Spinner.new("Deleting Pipeline #{model} ".ljust(JUSTIFICATION) + ":spinner", format: :dots)
+          spinner.auto_spin
+
+          unless model.exists?
+            spinner.stop(Rainbow("[MISSING]").white)
+            next
+          end
+
+          response = model.delete!
+          status = if response['acknowledged']
+            Rainbow("[SUCCESS]").green
+          else
+            Rainbow("[FAILED]").red
+          end
+          spinner.stop(status)
+        end
+      end
+  end
+end

data/lib/stretchy/rails/tasks/pipeline/status.rake

@@ -0,0 +1,25 @@
+namespace :stretchy do
+  namespace :pipeline do
+
+    desc "Check the status of all pipelines"
+    task status: :environment do
+      klass = ENV['MODEL']
+
+      Rails.application.eager_load!
+      models = klass.nil? ? Stretchy::Pipeline.descendants : [klass.constantize]
+
+      puts Rainbow("Pipelines").color :gray
+      models.each do |model|
+        response = model.exists?
+        status = if response
+          Rainbow("[CREATED]").green.bright
+        else
+          Rainbow("[MISSING]").white
+        end
+        puts "#{model.pipeline_name}".ljust(JUSTIFICATION) + status
+      end
+      puts "\n\n"
+    end
+
+  end
+end

data/lib/stretchy/rails/tasks/status.rake

@@ -0,0 +1,15 @@
+namespace :stretchy do
+  namespace :status do
+    desc "Check the status of all indexes"
+    task all: :environment do
+      Rake::Task['stretchy:index:status'].invoke
+      Rake::Task['stretchy:pipeline:status'].invoke
+      Rake::Task['stretchy:ml:status'].invoke
+      Rake::Task['stretchy:connector:status'].invoke
+    end
+  end
+
+  desc "Check the status of all indexes, pipelines and ml models"
+  task status: 'stretchy:status:all' do
+  end
+end

data/lib/stretchy/rails/tasks/stretchy.rake

@@ -0,0 +1,42 @@
+require 'stretchy-model'
+require 'tty-spinner'
+
+JUSTIFICATION = 90 unless defined?(JUSTIFICATION)
+
+path = File.expand_path(__dir__)
+Dir.glob("#{path}/**/*.rake").each do |f|
+  import f unless f == __FILE__
+end
+
+namespace :stretchy do
+  desc "Create all indexes, pipelines and deploy all models"
+  task up: :environment do
+    Rake::Task['stretchy:connector:create'].invoke
+    Rake::Task['stretchy:ml:register'].invoke
+    Rake::Task['stretchy:ml:deploy'].invoke
+    Rake::Task['stretchy:pipeline:create'].invoke
+    Rake::Task['stretchy:index:create'].invoke
+  end
+
+  desc "Delete all indexes, pipelines and undeploy all models"
+  task down: :environment do
+    Rake::Task['stretchy:ml:undeploy'].invoke
+    Rake::Task['stretchy:ml:delete'].invoke
+    Rake::Task['stretchy:connector:delete'].invoke
+    Rake::Task['stretchy:pipeline:delete'].invoke
+    Rake::Task['stretchy:index:delete'].invoke
+  end
+
+  desc "Enable Machine Learning on all nodes"
+  task ml_on_all_nodes: :environment do
+    puts "Enabling Machine Learning on all nodes..."
+    puts Stretchy::MachineLearning::Model.ml_on_all_nodes!
+  end
+
+  desc "Machine Learning on ML nodes only"
+  task ml_on_ml_nodes: :environment do
+    puts "Enabling Machine Learning on ML nodes only..."
+    puts Stretchy::MachineLearning::Model.ml_on_ml_nodes!
+  end
+
+end

data/lib/stretchy/version.rb

@@ -1,3 +1,3 @@
 module Stretchy
-  VERSION = '0.6.6'
+  VERSION = '0.7.0'
 end

data/lib/stretchy.rb

@@ -10,7 +10,9 @@ require 'active_support/all'
 
 require_relative "stretchy/version"
 require_relative "stretchy/rails/instrumentation/railtie" if defined?(Rails)
+require_relative "stretchy/rails/railtie" if defined?(Rails)
 require_relative 'elasticsearch/api/namespace/machine_learning/model'
+require_relative 'elasticsearch/api/namespace/connector'
 
 module Stretchy
 
@@ -27,6 +29,7 @@ module Stretchy
 
   def self.boot!
     loader.setup
+    Elasticsearch::API.send(:include, Elasticsearch::API::Connector)
     Elasticsearch::API.send(:include, Elasticsearch::API::MachineLearning::Models)
     Stretchy::Attributes.register!
   end
@@ -35,6 +38,10 @@ module Stretchy
     Stretchy::Delegation::GatewayDelegation.send(:include, Rails::Instrumentation::Publishers::Record)
   end
 
+  def self.env
+    ENV['RACK_ENV'] || ENV['RAILS_ENV'] || 'development'
+  end
+
   module Errors
     class QueryOptionMissing < StandardError; end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: stretchy-model
 version: !ruby/object:Gem::Version
-  version: 0.6.6
+  version: 0.7.0
 platform: ruby
 authors:
 - Spencer Markowski
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-03-28 00:00:00.000000000 Z
+date: 2024-03-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: zeitwerk
@@ -122,6 +122,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: tty-spinner
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -258,7 +272,7 @@ files:
 - docs/_sidebar.md
 - docs/examples/_sidebar.md
 - docs/examples/data_analysis.md
-- docs/examples/semantic_search_with_llm.md
+- docs/examples/neural_search_with_llm.md
 - docs/examples/simple-ingest-pipeline.md
 - docs/guides/_sidebar.md
 - docs/guides/aggregations.md
@@ -272,6 +286,24 @@ files:
 - docs/stretchy.cover.png
 - docs/stretchy.logo.png
 - docs/styles.css
+- lib/elasticsearch/api/actions/connector/check_in.rb
+- lib/elasticsearch/api/actions/connector/delete.rb
+- lib/elasticsearch/api/actions/connector/get.rb
+- lib/elasticsearch/api/actions/connector/last_sync.rb
+- lib/elasticsearch/api/actions/connector/list.rb
+- lib/elasticsearch/api/actions/connector/post.rb
+- lib/elasticsearch/api/actions/connector/put.rb
+- lib/elasticsearch/api/actions/connector/update_api_key_id.rb
+- lib/elasticsearch/api/actions/connector/update_configuration.rb
+- lib/elasticsearch/api/actions/connector/update_error.rb
+- lib/elasticsearch/api/actions/connector/update_filtering.rb
+- lib/elasticsearch/api/actions/connector/update_index_name.rb
+- lib/elasticsearch/api/actions/connector/update_name.rb
+- lib/elasticsearch/api/actions/connector/update_native.rb
+- lib/elasticsearch/api/actions/connector/update_pipeline.rb
+- lib/elasticsearch/api/actions/connector/update_scheduling.rb
+- lib/elasticsearch/api/actions/connector/update_service_type.rb
+- lib/elasticsearch/api/actions/connector/update_status.rb
 - lib/elasticsearch/api/actions/machine_learning/models/delete_model.rb
 - lib/elasticsearch/api/actions/machine_learning/models/deploy.rb
 - lib/elasticsearch/api/actions/machine_learning/models/get_model.rb
@@ -280,15 +312,23 @@ files:
 - lib/elasticsearch/api/actions/machine_learning/models/register.rb
 - lib/elasticsearch/api/actions/machine_learning/models/undeploy.rb
 - lib/elasticsearch/api/actions/machine_learning/models/update_model.rb
+- lib/elasticsearch/api/namespace/connector.rb
 - lib/elasticsearch/api/namespace/machine_learning/model.rb
+- lib/opensearch/api/actions/machine_learning/connector/delete.rb
+- lib/opensearch/api/actions/machine_learning/connector/get.rb
+- lib/opensearch/api/actions/machine_learning/connector/list.rb
+- lib/opensearch/api/actions/machine_learning/connector/post.rb
+- lib/opensearch/api/actions/machine_learning/connector/put.rb
 - lib/opensearch/api/actions/machine_learning/models/delete_model.rb
 - lib/opensearch/api/actions/machine_learning/models/deploy.rb
 - lib/opensearch/api/actions/machine_learning/models/get_model.rb
 - lib/opensearch/api/actions/machine_learning/models/get_status.rb
 - lib/opensearch/api/actions/machine_learning/models/params_registry.rb
+- lib/opensearch/api/actions/machine_learning/models/predict.rb
 - lib/opensearch/api/actions/machine_learning/models/register.rb
 - lib/opensearch/api/actions/machine_learning/models/undeploy.rb
 - lib/opensearch/api/actions/machine_learning/models/update_model.rb
+- lib/opensearch/api/namespace/connector.rb
 - lib/opensearch/api/namespace/machine_learning/model.rb
 - lib/stretchy.rb
 - lib/stretchy/associations.rb
@@ -348,7 +388,10 @@ files:
 - lib/stretchy/delegation/gateway_delegation.rb
 - lib/stretchy/index_setting.rb
 - lib/stretchy/indexing/bulk.rb
+- lib/stretchy/machine_learning/connector.rb
+- lib/stretchy/machine_learning/errors.rb
 - lib/stretchy/machine_learning/model.rb
+- lib/stretchy/machine_learning/registry.rb
 - lib/stretchy/model/callbacks.rb
 - lib/stretchy/model/common.rb
 - lib/stretchy/model/persistence.rb
@@ -360,6 +403,22 @@ files:
 - lib/stretchy/querying.rb
 - lib/stretchy/rails/instrumentation/publishers.rb
 - lib/stretchy/rails/instrumentation/railtie.rb
+- lib/stretchy/rails/railtie.rb
+- lib/stretchy/rails/tasks/connector/create.rake
+- lib/stretchy/rails/tasks/connector/delete.rake
+- lib/stretchy/rails/tasks/connector/status.rake
+- lib/stretchy/rails/tasks/connector/update.rake
+- lib/stretchy/rails/tasks/index/create.rake
+- lib/stretchy/rails/tasks/index/delete.rake
+- lib/stretchy/rails/tasks/index/status.rake
+- lib/stretchy/rails/tasks/ml/delete.rake
+- lib/stretchy/rails/tasks/ml/deploy.rake
+- lib/stretchy/rails/tasks/ml/status.rake
+- lib/stretchy/rails/tasks/pipeline/create.rake
+- lib/stretchy/rails/tasks/pipeline/delete.rake
+- lib/stretchy/rails/tasks/pipeline/status.rake
+- lib/stretchy/rails/tasks/status.rake
+- lib/stretchy/rails/tasks/stretchy.rake
 - lib/stretchy/record.rb
 - lib/stretchy/relation.rb
 - lib/stretchy/relations/aggregation_methods.rb

data/docs/examples/semantic_search_with_llm.md

@@ -1,83 +0,0 @@
-# Semantic Search with LLMs
-
->[!INFO|style:flat|label:OpenSearch Only]
-> This guide only works with features found in Opensearch 2.12+
-
-**Prerequisites:**
-
-- Opensearch installed and running
-- Ruby on Rails or stretchy-model's `bin/console`
-
-Follow the [Quick Start](guides/quick-start) for detailed steps. 
-
-## Text Embeddings
-
-* OpenAI embeddings
-
-## Ingest Pipeline
-
-_Set up ingest pipeline with inference model_
-
-## Define Models
-```ruby
-class Repo < StretchyModel
-	attribute :name, :string
-	attribute :meta, :hash
-	attribute :files, :nested
-	has_many :files
-end
-```
-
-```ruby
-class RepoFile < StretchyModel
-	attribute :name, :string
-    attribute :path, :string
-	attribute :content, :text
-	attribute :file_embeddings, :dense_vector
-	attribute :method_definitions, :array
-	attribute :language, :string
-end
-```
-
-## Controller
-```ruby
-def search
-  query_text = params[:query_text] 
-
-  response = RepoFile.where(content: query_text)
-        .exists?(field: :file_embeddings)
-        .knn(
-          field: :file_embeddings,
-          k: 1,
-          num_candidates: 20,
-          query_vector_builder: {
-            text_embedding: {
-                model_id: :sentence_transformers,
-                model_text: query_text
-            }
-          },
-          boost: 24
-        )
-        .fields(:content, :name, :path)
-
-
-  negative_response = "Unable to find the answer to your question given the provided context."
-
-  prompt = "Answer this question: #{query_text}\nUsing only the information from this Elastic Doc: #{response}\nIf the answer is not contained in the supplied doc reply '#{negative_response}' and nothing else"
-
-  @answer = LLM.chat(prompt)
-end
-```
-
-## Views
-
-```slim
-
-div[data-controller="search"]
-  = simple_form :search do |f|
-    = f.input :query_text
-    = f.submit
-
-#responses data-controller="response"
-
-```