stretchy-model 0.6.6 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ff8d7515eb6f795cf92989ba14d3dd404bb716378896ca2da4054bca98e467f4
-  data.tar.gz: 2361d39fcacb927f7d62bfe2105054c0923cd6b20af3cd7724c2bc9d784d42e5
+  metadata.gz: a741abc938cc22041b3aa0931f0a390b87262878402020fa095cbc47b35b6365
+  data.tar.gz: 87e565ac9cf56e341ff3541492d2ae32ba01a391ef745a74390789531e5a0cb9
 SHA512:
-  metadata.gz: 410eb69810533c4a12fe12b1bdd5ea6df9b320f9c1953a32f924487b02f1825296765e5a5451abf1c22491a6d9e8a1741e4e7ef01789acbac0026e1db423602d
-  data.tar.gz: 9afdb5b058a307c3b64f8043f1e4f2348973050b6b5c802726031f060a44ac11003633ba816ab278ba204c1a4946a12f0c0c7171fd72cf0188493c43b3e0d4df
+  metadata.gz: 72ac4716758a0b393f65cb928ecb7963b16f95c52865acba04f26b4356f5e0cb24d468d13d1f1f7e6aba6c5a8bc32c3db232796ab09c20136b174f9575934ae8
+  data.tar.gz: 577e03534c365c9746d3ddf1eba3263397752e5265d868268e78c1fc9851a490f31d953ace7ba7ab113d65493dc32b918720b75f5c13ad30a32d31f0597388eb

data/docs/_sidebar.md

@@ -11,4 +11,5 @@
 
 * __Examples__
     * [Data Analysis](examples/data_analysis)
-    * [Simple Ingest Pipeline](examples/simple-ingest-pipeline)
+    * [Simple Ingest Pipeline](examples/simple-ingest-pipeline)
+    * [Neural Search with LLM](examples/neural_search_with_llm)

data/docs/examples/_sidebar.md

@@ -12,4 +12,4 @@
 * __Examples__
     * [Data Analysis](examples/data_analysis)
     * [Simple Ingest Pipeline](examples/simple-ingest-pipeline?id=simple-ingest-pipeline)
-    * [Semantic Search with LLMs](examples/semantic_search_with_llm)
+    * [Neural Search with LLM](examples/neural_search_with_llm?id=neural-search-with-llm-expert)

data/docs/examples/neural_search_with_llm.md

@@ -0,0 +1,381 @@
+# Neural Search with LLM Expert
+
+This guide provides a comprehensive walkthrough on creating a Retrieval-Augmented Generation (RAG) expert system utilizing stretchy-model, OpenSearch, and OpenAI. By integrating these technologies, you'll develop an application capable of understanding and answering complex questions with context derived from its own data.
+
+The process involves several key steps:
+
+- **Environment Setup:** Initiating a Rails application and integrating necessary gems.
+- **Configuration:** Setting credentials for OpenSearch and OpenAI.
+- **Model and Pipeline Creation:** Defining models and pipelines for handling and indexing data.
+- **Machine Learning Integration:** Leveraging OpenAI's GPT model for generating responses based on the retrieved context.
+- **Deployment:** Deploying models and verifying the setup.
+- **Ingestion:** Ingesting a git repository.
+- **Interaction:** Asking questions about the data.
+
+## Create Rails Application
+
+```
+rails new stretchy-rails-app
+```
+
+```
+cd stretchy-rails-app
+```
+
+```
+bundle add opensearch-ruby stretchy-model github-linguist
+```
+
+### Set Credentials
+
+```
+rails credentials:edit
+```
+
+```yaml
+opensearch: 
+	host: https://localhost:9200 
+	user: admin 
+	password: admin 
+	transport_options: 
+		ssl: 
+		 verify: false
+
+openai:
+    openAI_key: <open_ai_key>
+```
+
+### Initialize Stretchy
+
+_config/initializers/stretchy.rb_ 
+```ruby
+Stretchy.configure do |config| 
+	config.client = OpenSearch::Client.new Rails.application.credentials.opensearch 
+end
+```
+
+### Start OpenSearch
+
+>[!WARNING]
+>This example deploys multiple Machine Learning models and will need a node for each. Follow the [multi-node docker instructions](https://gist.github.com/esmarkowski/7f3ec9bfb3b0dc3604112b67410067e7) to ensure you can fully deploy by the end of the example. 
+>
+
+```
+docker-compose -f opensearch/ml-compose.yml up
+```
+
+## Define Models
+
+### RepoFile
+
+*app/models/repo_file.rb*
+```ruby
+class RepoFile < StretchyModel
+	attribute :content, :text
+	attribute :file_name, :string
+	attribute :embeddings, :knn_vector, dimension: 384
+	
+	default_pipeline :text_embedding_pipeline
+	
+	index_settings(
+		'knn.space_type': :cosinesimil,
+		knn: true
+	)
+
+end
+```
+
+
+### Text Embedding Model
+
+*app/machine_learning/text_embedding_model.rb*
+```ruby
+class TextEmbeddingModel < Stretchy::MachineLearning::Model
+
+  model :sentence_transformers_minilm_12
+  model_format 'TORCH_SCRIPT'
+  version '1.0.1'
+
+end
+```
+
+## Ingest Pipeline 
+
+### Text Embedding Pipeline
+
+*app/pipelines/ingest/text_embedding_pipeline.rb*
+```ruby
+module Ingest
+  class TextEmbeddingPipeline < Stretchy::Pipeline
+
+    description "KNN text embedding pipeline"
+
+    processor :text_embedding, 
+      field_map: {
+        content: :embeddings
+      },
+      model: TextEmbeddingModel
+
+  end
+end
+```
+
+## Configure LLM 
+
+## Create a connector
+
+```ruby
+module Connectors
+  class GPTConnector < Stretchy::MachineLearning::Connector
+
+    description "The connector to OpenAI's gpt-3.5-turbo service for gpt model"
+      
+    version 1
+    
+    protocol "http"
+    
+    credentials Rails.application.credentials.dig(:openai)
+    
+    parameters endpoint: "api.openai.com",
+               model: 'gpt-3.5-turbo'
+
+    actions action_type: "predict",
+            method: "POST",
+            url: "https://${parameters.endpoint}/v1/chat/completions",
+            headers: {
+              "Authorization": "Bearer ${credential.openAI_key}"
+            },
+            request_body: "{\"model\":\"${parameters.model}\",\"messages\": ${parameters.messages}}"
+       
+  end
+end
+```
+
+### Create an LLM Model
+
+```ruby
+module Models
+	class GPT < Stretchy::MachineLearning::Model
+	
+	  model_name 'gpt'
+	  function_name :remote
+	  connector 'Connectors::GPTConnector'
+	
+	  def self.predict(prompt)
+	    response = client.predict(model_id: self.model_id, body: prompt)
+		response.dig('inference_results')
+			.first.dig('output')
+			.first.dig('dataAsMap', 'choices')
+			.first.dig('message', 'content')
+	  end
+	end
+end
+```
+
+### Define an expert
+
+```ruby
+class Expert
+
+	BEHAVIOR = "You are an expert in Ruby, Ruby on Rails, Elasticsearch and Opensearch. You read documentation and provide succinct and direct answers to the questions provided using the context provided. If the answer is not directly shown in the context, you will analyze the data and find the answer. If you don't know the answer, just say you don't know."
+
+	def get_context(question, k=2)
+		RepoFile.neural(
+			embeddings: question,
+			model_id: TextEmbeddingModel.model_id,
+			k: k
+		).pluck(:content)
+	end
+	
+	def ask(question)
+		prompt = {
+			parameters: {
+				messages: [
+					{
+						"role": "system",
+						"content": BEHAVIOR
+					},
+					{
+						"role": "assistant",
+						"content": get_context(question).join("\n")
+					},
+					{
+						"role": "user",
+						"content": question
+					}
+				]
+			}
+		}
+		
+		Models::GPT.predict(prompt)
+	end
+end
+
+```
+
+## Deploy Models
+
+Stretchy includes rake tasks to aid in managing your resources. Running `rake stretchy:up` will ensure dependencies are handled by performing the following steps:
+- Create `Connectors`
+- Register and deploy `MachineLearning::Models`
+- Create `Pipelines` 
+- Create indexes for all `StretchyModels`
+
+> [!INFO|style:flat|label:Machine Learning Nodes]
+> If you do not have dedicated machine learning nodes (or are running a single-node cluster) you'll need to enable machine learning on all nodes. 
+> 
+> ```
+> rake stretchy:ml_on_all_nodes
+> ```
+> 
+
+Run the following to start the deployment: 
+
+```
+rake stretchy:up
+```
+
+![[docs/media/stretchy_up.mov]]
+
+>[!TIP]
+>Registering and deploying machine learning models can take some time. 
+>Once it's complete you can confirm the status with `rake stretchy:status`
+>
+> If you'd like to explore other rake tasks available run `rake -T | grep stretchy`
+
+
+## Create a Source
+We create a simple source that pulls a git repository and allows us to index it into `RepoFile`. 
+
+*app/models/sources/git.rb*
+```ruby
+require 'open3'
+
+module Sources
+  class Git
+
+    attr_reader :repo_url, :repo_name, :path
+    attr_accessor :errors
+
+    def initialize(repo_url, path: '/tmp')
+      @repo_url = repo_url
+      @repo_name = extract_repo_name(repo_url)
+      @path = path
+    end
+
+    # Perform the git clone and ingest
+    def perform(&block)
+      clone unless File.directory?("#{path}/#{repo_name}")
+      ingest(repo_url, &block)
+    end
+
+    # Clone the repo to path 
+    def clone
+      clone_cmd = "git clone #{repo_url} #{path}/#{repo_name}"
+      Open3.popen2e(clone_cmd) do |stdin, stdout_err, wait_thr|
+        while line = stdout_err.gets
+          Rails.logger.debug line
+        end
+        exit_status = wait_thr.value
+        unless exit_status.success?
+          raise "FAILED to clone #{repo_url}"
+        end
+      end
+    end
+
+
+    # Remove the repo from the path
+    def remove
+      FileUtils.rm_rf("#{path}/#{repo_name}")
+    end
+
+    # Recursively crawl each file in the repo and yield the file to the block
+    def ingest(repo_url, &block)
+      @errors = []
+      Dir.glob("#{path}/#{repo_name}/**/*").each do |file|
+        if File.file?(file)
+          begin
+            yield file, self if block_given?
+          rescue => e
+            errors << [file, e]
+          end
+        end
+      end
+      
+      return true
+    end
+
+    def errors?
+      errors.any?
+    end
+
+    private
+
+    def extract_repo_name(url)
+      url.split('/').last.gsub('.git', '')
+    end
+
+  end
+end
+
+```
+
+## Ingest Data
+
+Start Rails console with `rails console` and run the following code.
+
+This will checkout the repo and bulk index the file contents into `RepoFile`, automatically using the `TextEmbeddingPipeline` to create embeddings during ingest. 
+
+```ruby
+source = Sources::Git.new('https://github.com/theablefew/stretchy.git')
+
+repo_files = source.perform do |file, instance|
+	next unless ['.md', '.rb'].include?(File.extname(file))
+	
+	RepoFile.new(
+		content: File.read(file),
+		file_name: File.basename(file),
+	)
+end.compact
+
+RepoFile.bulk_in_batches(repo_files, size: 100) do |batch|
+	batch.map! { |record | record.to_bulk }
+end
+```
+
+
+> [!TIP|label:Test it out]
+> 
+> We can now perform semantic searches using `neural` search. 
+> 
+> ```ruby
+> RepoFile.neural(
+>   embeddings: "How do I perform a percentile rank aggregation?",
+>   model_id: TextEmbeddingModel.model_id,
+>   k: 2
+> )
+> ```
+> This is how our `Expert` will gather relevant context to pass to `GPT` LLM. 
+
+## Interact
+
+```ruby
+expert = Expert.new
+expert.ask("How can I perform a percentile ranks aggregation?")
+```
+
+We’ve asked how to perform a percentile ranks aggregation and provided our `Expert` relevant documentation as context with a `neural` search. 
+
+> [!INFO|label:LLM Response]
+>  You can perform a percentile ranks aggregation in Stretchy by using the `percentile_ranks` method. Here is an example to calculate the percentile ranks for values `[1, 2, 3]` on the field `'field_name'`:
+> 
+> ```ruby
+> Model.percentile_ranks(:my_agg, {field: 'field_name', values: [1, 2, 3]})
+> ```
+>
+>  This method will calculate the percentile ranks for the specified values on the specified field. 
+
+
+### Conclusion
+
+By following this guide, you've created a powerful expert system capable of providing informed responses to complex questions. This integration not only demonstrates the potential of combining semantic search with generative AI but also provides a solid foundation for further exploration and development.

data/docs/guides/_sidebar.md

@@ -11,4 +11,5 @@
 
 * __Examples__
     * [Data Analysis](examples/data_analysis)
-    * [Simple Ingest Pipeline](examples/simple-ingest-pipeline)
+    * [Simple Ingest Pipeline](examples/simple-ingest-pipeline)
+    * [Neural Search with LLM](examples/neural_search_with_llm)

data/lib/elasticsearch/api/actions/connector/check_in.rb

@@ -0,0 +1,64 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Auto generated from build hash f284cc16f4d4b4289bc679aa1529bb504190fe80
+# @see https://github.com/elastic/elasticsearch/tree/main/rest-api-spec
+#
+module Elasticsearch
+  module API
+    module Connector
+      module Actions
+        # Updates the last_seen timestamp in the connector document.
+        # This functionality is Experimental and may be changed or removed
+        # completely in a future release. Elastic will take a best effort approach
+        # to fix any issues, but experimental features are not subject to the
+        # support SLA of official GA features.
+        #
+        # @option arguments [String] :connector_id The unique identifier of the connector to be updated.
+        # @option arguments [Hash] :headers Custom HTTP headers
+        #
+        # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/check-in-connector-api.html
+        #
+        def check_in(arguments = {})
+          request_opts = { endpoint: arguments[:endpoint] || 'connector.check_in' }
+
+          defined_params = [:connector_id].each_with_object({}) do |variable, set_variables|
+            set_variables[variable] = arguments[variable] if arguments.key?(variable)
+          end
+          request_opts[:defined_params] = defined_params unless defined_params.empty?
+
+          raise ArgumentError, "Required argument 'connector_id' missing" unless arguments[:connector_id]
+
+          arguments = arguments.clone
+          headers = arguments.delete(:headers) || {}
+
+          body = nil
+
+          _connector_id = arguments.delete(:connector_id)
+
+          method = Elasticsearch::API::HTTP_PUT
+          path   = "_connector/#{Utils.__listify(_connector_id)}/_check_in"
+          params = {}
+
+          Elasticsearch::API::Response.new(
+            perform_request(method, path, params, body, headers, request_opts)
+          )
+        end
+      end
+    end
+  end
+end

data/lib/elasticsearch/api/actions/connector/delete.rb

@@ -0,0 +1,64 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Auto generated from build hash f284cc16f4d4b4289bc679aa1529bb504190fe80
+# @see https://github.com/elastic/elasticsearch/tree/main/rest-api-spec
+#
+module Elasticsearch
+  module API
+    module Connector
+      module Actions
+        # Deletes a connector.
+        # This functionality is Experimental and may be changed or removed
+        # completely in a future release. Elastic will take a best effort approach
+        # to fix any issues, but experimental features are not subject to the
+        # support SLA of official GA features.
+        #
+        # @option arguments [String] :connector_id The unique identifier of the connector to be deleted.
+        # @option arguments [Hash] :headers Custom HTTP headers
+        #
+        # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/delete-connector-api.html
+        #
+        def delete(arguments = {})
+          request_opts = { endpoint: arguments[:endpoint] || 'connector.delete' }
+
+          defined_params = [:connector_id].each_with_object({}) do |variable, set_variables|
+            set_variables[variable] = arguments[variable] if arguments.key?(variable)
+          end
+          request_opts[:defined_params] = defined_params unless defined_params.empty?
+
+          raise ArgumentError, "Required argument 'connector_id' missing" unless arguments[:connector_id]
+
+          arguments = arguments.clone
+          headers = arguments.delete(:headers) || {}
+
+          body = nil
+
+          _connector_id = arguments.delete(:connector_id)
+
+          method = Elasticsearch::API::HTTP_DELETE
+          path   = "_connector/#{Utils.__listify(_connector_id)}"
+          params = {}
+
+          Elasticsearch::API::Response.new(
+            perform_request(method, path, params, body, headers, request_opts)
+          )
+        end
+      end
+    end
+  end
+end

data/lib/elasticsearch/api/actions/connector/get.rb

@@ -0,0 +1,64 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Auto generated from build hash f284cc16f4d4b4289bc679aa1529bb504190fe80
+# @see https://github.com/elastic/elasticsearch/tree/main/rest-api-spec
+#
+module Elasticsearch
+  module API
+    module Connector
+      module Actions
+        # Returns the details about a connector.
+        # This functionality is Experimental and may be changed or removed
+        # completely in a future release. Elastic will take a best effort approach
+        # to fix any issues, but experimental features are not subject to the
+        # support SLA of official GA features.
+        #
+        # @option arguments [String] :connector_id The unique identifier of the connector to be returned.
+        # @option arguments [Hash] :headers Custom HTTP headers
+        #
+        # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/get-connector-api.html
+        #
+        def get(arguments = {})
+          request_opts = { endpoint: arguments[:endpoint] || 'connector.get' }
+
+          defined_params = [:connector_id].each_with_object({}) do |variable, set_variables|
+            set_variables[variable] = arguments[variable] if arguments.key?(variable)
+          end
+          request_opts[:defined_params] = defined_params unless defined_params.empty?
+
+          raise ArgumentError, "Required argument 'connector_id' missing" unless arguments[:connector_id]
+
+          arguments = arguments.clone
+          headers = arguments.delete(:headers) || {}
+
+          body = nil
+
+          _connector_id = arguments.delete(:connector_id)
+
+          method = Elasticsearch::API::HTTP_GET
+          path   = "_connector/#{Utils.__listify(_connector_id)}"
+          params = {}
+
+          Elasticsearch::API::Response.new(
+            perform_request(method, path, params, body, headers, request_opts)
+          )
+        end
+      end
+    end
+  end
+end

data/lib/elasticsearch/api/actions/connector/last_sync.rb

@@ -0,0 +1,66 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Auto generated from build hash f284cc16f4d4b4289bc679aa1529bb504190fe80
+# @see https://github.com/elastic/elasticsearch/tree/main/rest-api-spec
+#
+module Elasticsearch
+  module API
+    module Connector
+      module Actions
+        # Updates the stats of last sync in the connector document.
+        # This functionality is Experimental and may be changed or removed
+        # completely in a future release. Elastic will take a best effort approach
+        # to fix any issues, but experimental features are not subject to the
+        # support SLA of official GA features.
+        #
+        # @option arguments [String] :connector_id The unique identifier of the connector to be updated.
+        # @option arguments [Hash] :headers Custom HTTP headers
+        # @option arguments [Hash] :body Object with stats related to the last connector sync run. (*Required*)
+        #
+        # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/update-connector-last-sync-api.html
+        #
+        def last_sync(arguments = {})
+          request_opts = { endpoint: arguments[:endpoint] || 'connector.last_sync' }
+
+          defined_params = [:connector_id].each_with_object({}) do |variable, set_variables|
+            set_variables[variable] = arguments[variable] if arguments.key?(variable)
+          end
+          request_opts[:defined_params] = defined_params unless defined_params.empty?
+
+          raise ArgumentError, "Required argument 'body' missing" unless arguments[:body]
+          raise ArgumentError, "Required argument 'connector_id' missing" unless arguments[:connector_id]
+
+          arguments = arguments.clone
+          headers = arguments.delete(:headers) || {}
+
+          body = arguments.delete(:body)
+
+          _connector_id = arguments.delete(:connector_id)
+
+          method = Elasticsearch::API::HTTP_PUT
+          path   = "_connector/#{Utils.__listify(_connector_id)}/_last_sync"
+          params = {}
+
+          Elasticsearch::API::Response.new(
+            perform_request(method, path, params, body, headers, request_opts)
+          )
+        end
+      end
+    end
+  end
+end