langchainrb 0.6.0 → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7fba7b5e03ae75aa4ee3c89dd0322a73bbb9d3ced79f48dda8861af1f4f197b8
-  data.tar.gz: 404e742b0911305beec3bd22575740fc78ed9005e21295e0f7c348c1bede3e7e
+  metadata.gz: 5a6f4e8bb8ecaba6ff4d53bba384bd6338012429a69a0dc7df0a58a476763e7e
+  data.tar.gz: 92211a22fca9664831cf4f395a53dedddafc339ab419780932398c07256b737d
 SHA512:
-  metadata.gz: c8166375c28abe9bc3a7e02a2ceba2ce1bea8ff4f751beb2c69d6f34aa46ba2b7c6ca34458f79b6dfba6d8908cc5e9b055f4e4e7dc6c4c09ac65f7f589c12eb7
-  data.tar.gz: fb81a51867575c5fae10b79f0f3ee761b25b75d623f2c071d000b39f84c672d9f7164e3c8442a1f11d7abdff8edd41f7e79c2c3a7cb99fa9952406bc60506ce8
+  metadata.gz: b5c84f0a9a54f51799c5318cba243457fcfd6f026c71b8f34e58cf60172d476963f25ea8d24c49b35ed93c893adb9e2844443a22dd9e927ab16318850a11419a
+  data.tar.gz: 4664927203ea032f737000c27ec5fa04c96ab606ec8377b4673b48638905b458077d4ab3cb7727fcb98be6c607a37bd318395fd96000a734de213c7d9041a219

data/CHANGELOG.md

@@ -1,5 +1,8 @@
 ## [Unreleased]
 
+## [0.6.1] - 2023-06-24
+- Adding support to hook vectorsearch into ActiveRecord models
+
 ## [0.6.0] - 2023-06-22
 - [BREAKING] Rename `ChainOfThoughtAgent` to `ReActAgent`
 - Implement A21 token validator

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    langchainrb (0.6.0)
+    langchainrb (0.6.1)
       baran (~> 0.1.6)
       colorize (~> 0.8.1)
       json-schema (~> 4.0.0)

data/README.md

@@ -41,7 +41,7 @@ require "langchain"
 | [Pinecone](https://www.pinecone.io/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
 | [Pgvector](https://github.com/pgvector/pgvector) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
 | [Qdrant](https://qdrant.tech/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
-| [Weaviate](https://weaviate.io/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | WIP               |
+| [Weaviate](https://weaviate.io/) | :white_check_mark: | :white_check_mark: | :white_check_mark: | WIP     | :white_check_mark: |
 
 ### Using Vector Search Databases 🔍
 

data/lib/langchain/active_record/hooks.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module Langchain
+  module ActiveRecord
+    # This module adds the following functionality to your ActiveRecord models:
+    # * `vectorsearch` class method to set the vector search provider
+    # * `similarity_search` class method to search for similar texts
+    # * `upsert_to_vectorsearch` instance method to upsert the record to the vector search provider
+    #
+    # Usage:
+    #     class Recipe < ActiveRecord::Base
+    #       vectorsearch provider: Langchain::Vectorsearch::Weaviate.new(
+    #                    api_key: ENV["WEAVIATE_API_KEY"],
+    #                    url: ENV["WEAVIATE_URL"],
+    #                    index_name: "Recipes",
+    #                    llm: Langchain::LLM::OpenAI.new(api_key: ENV["OPENAI_API_KEY"])
+    #                 )
+    #
+    #       after_save :upsert_to_vectorsearch
+    #
+    #       # Overwriting how the model is serialized before it's indexed
+    #       def as_vector
+    #         [
+    #           "Title: #{title}",
+    #           "Description: #{description}",
+    #           ...
+    #         ]
+    #         .compact
+    #         .join("\n")
+    #       end
+    #     end
+    #
+    # Create the default schema
+    #     Recipe.class_variable_get(:@@provider).create_default_schema
+    # Query the vector search provider
+    #     Recipe.similarity_search("carnivore dish")
+    # Delete the default schema to start over
+    #     Recipe.class_variable_get(:@@provider).client.schema.delete class_name: "Recipes"
+    #
+    module Hooks
+      def self.included(base)
+        base.extend ClassMethods
+      end
+
+      # Index the text to the vector search provider
+      # You'd typically call this method in an ActiveRecord callback
+      #
+      # @return [Boolean] true
+      # @raise [Error] Indexing to vector search DB failed
+      def upsert_to_vectorsearch
+        if previously_new_record?
+          self.class.class_variable_get(:@@provider).add_texts(
+            texts: [as_vector],
+            ids: [id]
+          )
+        else
+          self.class.class_variable_get(:@@provider).update_texts(
+            texts: [as_vector],
+            ids: [id]
+          )
+        end
+      end
+
+      # Used to serialize the DB record to an indexable vector text
+      # Overwrite this method in your model to customize
+      #
+      # @return [String] the text representation of the model
+      def as_vector
+        to_json
+      end
+
+      module ClassMethods
+        # Set the vector search provider
+        #
+        # @param provider [Object] The `Langchain::Vectorsearch::*` instance
+        def vectorsearch(provider:)
+          class_variable_set(:@@provider, provider)
+        end
+
+        # Search for similar texts
+        #
+        # @param query [String] The query to search for
+        # @param k [Integer] The number of results to return
+        # @return [ActiveRecord::Relation] The ActiveRecord relation
+        def similarity_search(query, k: 1)
+          records = class_variable_get(:@@provider).similarity_search(
+            query: query,
+            k: k
+          )
+          ids = records.map { |record| record.dig("__id") }
+          where(id: ids)
+        end
+      end
+    end
+  end
+end

data/lib/langchain/railtie.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Langchain
+  class Railtie < Rails::Railtie
+    initializer "langchain" do
+      ActiveSupport.on_load(:active_record) do
+        ::ActiveRecord::Base.include Langchain::ActiveRecord::Hooks
+      end
+    end
+  end
+end

data/lib/langchain/vectorsearch/weaviate.rb

@@ -14,7 +14,7 @@ module Langchain::Vectorsearch
     # Initialize the Weaviate adapter
     # @param url [String] The URL of the Weaviate instance
     # @param api_key [String] The API key to use
-    # @param index_name [String] The name of the index to use
+    # @param index_name [String] The capitalized name of the index to use
     # @param llm [Object] The LLM client to use
     def initialize(url:, api_key:, index_name:, llm:)
       depends_on "weaviate-ruby"
@@ -24,6 +24,9 @@ module Langchain::Vectorsearch
         url: url,
         api_key: api_key
       )
+
+      # Weaviate requires the class name to be Capitalized: https://weaviate.io/developers/weaviate/configuration/schema-configuration#create-a-class
+      # TODO: Capitalize index_name
       @index_name = index_name
 
       super(llm: llm)
@@ -32,31 +35,51 @@ module Langchain::Vectorsearch
     # Add a list of texts to the index
     # @param texts [Array] The list of texts to add
     # @return [Hash] The response from the server
-    def add_texts(texts:)
-      objects = Array(texts).map do |text|
-        {
-          class: index_name,
-          properties: {content: text},
-          vector: llm.embed(text: text)
-        }
-      end
-
+    def add_texts(texts:, ids:)
       client.objects.batch_create(
-        objects: objects
+        objects: weaviate_objects(texts, ids)
       )
     end
 
+    # Update a list of texts in the index
+    # @param texts [Array] The list of texts to update
+    # @return [Hash] The response from the server
+    def update_texts(texts:, ids:)
+      uuids = []
+
+      # Retrieve the UUIDs of the objects to update
+      Array(texts).map.with_index do |text, i|
+        record = client.query.get(
+          class_name: index_name,
+          fields: "_additional { id }",
+          where: "{ path: [\"__id\"], operator: Equal, valueString: \"#{ids[i]}\" }"
+        )
+        uuids.push record[0].dig("_additional", "id")
+      end
+
+      # Update the objects
+      texts.map.with_index do |text, i|
+        client.objects.update(
+          class_name: index_name,
+          id: uuids[i],
+          properties: {
+            __id: ids[i].to_s,
+            content: text
+          },
+          vector: llm.embed(text: text)
+        )
+      end
+    end
+
     # Create default schema
     def create_default_schema
       client.schema.create(
         class_name: index_name,
         vectorizer: "none",
         properties: [
-          # TODO: Allow passing in your own IDs
-          {
-            dataType: ["text"],
-            name: "content"
-          }
+          # __id to be used a pointer to the original document
+          {dataType: ["string"], name: "__id"}, # '_id' is a reserved property name (single underscore)
+          {dataType: ["text"], name: "content"}
         ]
       )
     end
@@ -82,7 +105,7 @@ module Langchain::Vectorsearch
         class_name: index_name,
         near_vector: near_vector,
         limit: k.to_s,
-        fields: "content _additional { id }"
+        fields: "__id content _additional { id }"
       )
     end
 
@@ -101,5 +124,24 @@ module Langchain::Vectorsearch
 
       llm.chat(prompt: prompt)
     end
+
+    private
+
+    def weaviate_objects(texts, ids)
+      Array(texts).map.with_index do |text, i|
+        weaviate_object(text, ids[i])
+      end
+    end
+
+    def weaviate_object(text, id)
+      {
+        class: index_name,
+        properties: {
+          __id: id.to_s,
+          content: text
+        },
+        vector: llm.embed(text: text)
+      }
+    end
   end
 end

data/lib/langchain/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Langchain
-  VERSION = "0.6.0"
+  VERSION = "0.6.1"
 end

data/lib/langchain.rb

@@ -145,6 +145,10 @@ module Langchain
     autoload :FewShotPromptTemplate, "langchain/prompt/few_shot_prompt_template"
   end
 
+  module ActiveRecord
+    autoload :Hooks, "langchain/active_record/hooks"
+  end
+
   module OutputParsers
     autoload :Base, "langchain/output_parsers/base"
     autoload :StructuredOutputParser, "langchain/output_parsers/structured"
@@ -154,3 +158,5 @@ module Langchain
     class BaseError < StandardError; end
   end
 end
+
+require "langchain/railtie" if defined?(Rails)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: langchainrb
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.6.1
 platform: ruby
 authors:
 - Andrei Bondarev
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-06-23 00:00:00.000000000 Z
+date: 2023-06-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: baran
@@ -483,6 +483,7 @@ files:
 - examples/store_and_query_with_weaviate.rb
 - lefthook.yml
 - lib/langchain.rb
+- lib/langchain/active_record/hooks.rb
 - lib/langchain/agent/base.rb
 - lib/langchain/agent/react_agent/react_agent.rb
 - lib/langchain/agent/react_agent/react_agent_prompt.yaml
@@ -519,6 +520,7 @@ files:
 - lib/langchain/prompt/few_shot_prompt_template.rb
 - lib/langchain/prompt/loading.rb
 - lib/langchain/prompt/prompt_template.rb
+- lib/langchain/railtie.rb
 - lib/langchain/tool/base.rb
 - lib/langchain/tool/calculator.rb
 - lib/langchain/tool/database.rb