langchainrb 0.6.8 → 0.6.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e8446cff4207c4784ce29233712430d50c2cbaa551cfe8ad27f413f27ad8346b
-  data.tar.gz: 8c7105302b673479be9a840ed1a19c815cf5a0415195db5373ce739c472dc2f3
+  metadata.gz: 4d046a7cae545da3092694fee60b0fa3caa0852492c9bf7dbbe35001d96106e4
+  data.tar.gz: 8ed841d2359400f9a4700a49b82030fbe3dd3563545eca31b487914e8fb236f7
 SHA512:
-  metadata.gz: d7bd52c8d161a2d26a1e717711053267ed3b78def7efa2e9411017d324e278baceb1d88eb709cb701986201b785e1ad8ee455948b1bba59d6cb478226b1b5df4
-  data.tar.gz: eeb639084a8990e1d36d035e6cd6d725377cecc61b3116f70254787f5d591a387c6f53e308dd482d2a96257f231866cffdf6326c516dcf78e60017817d8ebb93
+  metadata.gz: eed2f03cbdd74ef42fafd07524dc6bf5c7ac2278e5e55dd1aad5b2cd87f319a91c8719a6e9ac43ac50cda5525f3defab14cfecc749bdec81075d52b3935b23dd
+  data.tar.gz: cd4fce88b0f9f545f6d32660879e6fa5044ec74b41dcb41bc29eafb78b307d8b2cb8204856098657d8c9707331c8629c8a707a227fe4d20a5816347f8a66991c

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 ## [Unreleased]
 
+## [0.6.9] - 2023-07-29
+
+## [0.6.8] - 2023-07-21
+
 ## [0.6.7] - 2023-07-19
 - Support for OpenAI functions
 - Streaming vectorsearch ask() responses

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    langchainrb (0.6.8)
+    langchainrb (0.6.9)
       baran (~> 0.1.6)
       colorize (~> 0.8.1)
       json-schema (~> 4.0.0)
@@ -34,7 +34,7 @@ GEM
     afm (0.2.2)
     ai21 (0.2.1)
     ast (2.4.2)
-    baran (0.1.6)
+    baran (0.1.7)
     builder (3.2.4)
     byebug (11.1.3)
     childprocess (4.1.0)
@@ -189,7 +189,7 @@ GEM
       ruby-rc4
       ttfunk
     pg (1.5.3)
-    pgvector (0.2.0)
+    pgvector (0.2.1)
     pinecone (0.1.71)
       dry-struct (~> 1.6.0)
       dry-validation (~> 1.10.0)
@@ -334,7 +334,7 @@ DEPENDENCIES
   open-weather-ruby-client (~> 0.3.0)
   pdf-reader (~> 1.4)
   pg (~> 1.5)
-  pgvector (~> 0.2)
+  pgvector (~> 0.2.1)
   pinecone (~> 0.1.6)
   pry-byebug (~> 3.10.0)
   qdrant-ruby (~> 0.9.0)
@@ -353,4 +353,4 @@ DEPENDENCIES
   yard
 
 BUNDLED WITH
-   2.4.0
+   2.3.22

data/lib/langchain/utils/token_length/base_validator.rb

@@ -22,7 +22,7 @@ module Langchain
           leftover_tokens = token_limit(model_name) - text_token_length
 
           # Raise an error even if whole prompt is equal to the model's token limit (leftover_tokens == 0)
-          if leftover_tokens <= 0
+          if leftover_tokens < 0
             raise limit_exceeded_exception(token_limit(model_name), text_token_length)
           end
 

data/lib/langchain/utils/token_length/openai_validator.rb

@@ -25,6 +25,7 @@ module Langchain
           "code-davinci-002" => 8001,
           "gpt-4" => 8192,
           "gpt-4-0314" => 8192,
+          "gpt-4-0613" => 8192,
           "gpt-4-32k" => 32768,
           "gpt-4-32k-0314" => 32768,
           "gpt-4-32k-0613" => 32768,

data/lib/langchain/vectorsearch/pgvector.rb

@@ -8,38 +8,45 @@ module Langchain::Vectorsearch
     # Gem requirements: gem "pgvector", "~> 0.2"
     #
     # Usage:
-    # pgvector = Langchain::Vectorsearch::Pgvector.new(url:, index_name:, llm:, llm_api_key:)
+    # pgvector = Langchain::Vectorsearch::Pgvector.new(url:, index_name:, llm:, namespace_column: nil, namespace: nil)
     #
 
     # The operators supported by the PostgreSQL vector search adapter
     OPERATORS = {
-      "cosine_distance" => "<=>",
-      "euclidean_distance" => "<->"
+      "cosine_distance" => "cosine",
+      "euclidean_distance" => "euclidean"
     }
     DEFAULT_OPERATOR = "cosine_distance"
 
-    attr_reader :operator, :quoted_table_name
+    attr_reader :db, :operator, :table_name, :namespace_column, :namespace, :documents_table
 
     # @param url [String] The URL of the PostgreSQL database
     # @param index_name [String] The name of the table to use for the index
     # @param llm [Object] The LLM client to use
-    # @param api_key [String] The API key for the Vectorsearch DB (not used for PostgreSQL)
-    def initialize(url:, index_name:, llm:, api_key: nil)
-      require "pg"
+    # @param namespace [String] The namespace to use for the index when inserting/querying
+    def initialize(url:, index_name:, llm:, namespace: nil)
+      depends_on "sequel"
+      require "sequel"
+      depends_on "pgvector"
       require "pgvector"
 
-      @client = ::PG.connect(url)
-      registry = ::PG::BasicTypeRegistry.new.define_default_types
-      ::Pgvector::PG.register_vector(registry)
-      @client.type_map_for_results = PG::BasicTypeMapForResults.new(@client, registry: registry)
+      @db = Sequel.connect(url)
 
-      @index_name = index_name
-      @quoted_table_name = @client.quote_ident(index_name)
+      @table_name = index_name
+
+      @namespace_column = "namespace"
+      @namespace = namespace
       @operator = OPERATORS[DEFAULT_OPERATOR]
 
       super(llm: llm)
     end
 
+    def documents_model
+      Class.new(Sequel::Model(table_name.to_sym)) do
+        plugin :pgvector, :vectors
+      end
+    end
+
     # Upsert a list of texts to the index
     # @param texts [Array<String>] The texts to add to the index
     # @param ids [Array<Integer>] The ids of the objects to add to the index, in the same order as the texts
@@ -47,32 +54,28 @@ module Langchain::Vectorsearch
     # the added or updated texts.
     def upsert_texts(texts:, ids:)
       data = texts.zip(ids).flat_map do |(text, id)|
-        [id, text, llm.embed(text: text)]
+        {id: id, content: text, vectors: llm.embed(text: text).to_s, namespace: namespace}
       end
-      values = texts.length.times.map { |i| "($#{3 * i + 1}, $#{3 * i + 2}, $#{3 * i + 3})" }.join(",")
-      # see https://github.com/pgvector/pgvector#storing
-      client.exec_params(
-        "INSERT INTO #{quoted_table_name} (id, content, vectors) VALUES
-#{values} ON CONFLICT (id) DO UPDATE SET content = EXCLUDED.content, vectors = EXCLUDED.vectors RETURNING id;",
-        data
-      )
+      # @db[table_name.to_sym].multi_insert(data, return: :primary_key)
+      @db[table_name.to_sym]
+        .insert_conflict(
+          target: :id,
+          update: {content: Sequel[:excluded][:content], vectors: Sequel[:excluded][:vectors]}
+        )
+        .multi_insert(data, return: :primary_key)
     end
 
     # Add a list of texts to the index
     # @param texts [Array<String>] The texts to add to the index
     # @param ids [Array<String>] The ids to add to the index, in the same order as the texts
-    # @return [PG::Result] The response from the database including the ids of
-    # the added texts.
+    # @return [Array<Integer>] The the ids of the added texts.
     def add_texts(texts:, ids: nil)
       if ids.nil? || ids.empty?
-        data = texts.flat_map do |text|
-          [text, llm.embed(text: text)]
+        data = texts.map do |text|
+          {content: text, vectors: llm.embed(text: text).to_s, namespace: namespace}
         end
-        values = texts.length.times.map { |i| "($#{2 * i + 1}, $#{2 * i + 2})" }.join(",")
-        client.exec_params(
-          "INSERT INTO #{quoted_table_name} (content, vectors) VALUES #{values} RETURNING id;",
-          data
-        )
+
+        @db[table_name.to_sym].multi_insert(data, return: :primary_key)
       else
         upsert_texts(texts: texts, ids: ids)
       end
@@ -81,8 +84,7 @@ module Langchain::Vectorsearch
     # Update a list of ids and corresponding texts to the index
     # @param texts [Array<String>] The texts to add to the index
     # @param ids [Array<String>] The ids to add to the index, in the same order as the texts
-    # @return [PG::Result] The response from the database including the ids of
-    # the updated texts.
+    # @return [Array<Integer>] The ids of the updated texts.
     def update_texts(texts:, ids:)
       upsert_texts(texts: texts, ids: ids)
     end
@@ -90,16 +92,15 @@ module Langchain::Vectorsearch
     # Create default schema
     # @return [PG::Result] The response from the database
     def create_default_schema
-      client.exec("CREATE EXTENSION IF NOT EXISTS vector;")
-      client.exec(
-        <<~SQL
-          CREATE TABLE IF NOT EXISTS #{quoted_table_name} (
-            id serial PRIMARY KEY,
-            content TEXT,
-            vectors VECTOR(#{default_dimension})
-          );
-        SQL
-      )
+      db.run "CREATE EXTENSION IF NOT EXISTS vector"
+      namespace = namespace_column
+      vector_dimension = default_dimension
+      db.create_table? table_name.to_sym do
+        primary_key :id
+        text :content
+        column :vectors, "vector(#{vector_dimension})"
+        text namespace.to_sym, default: nil
+      end
     end
 
     # TODO: Add destroy_default_schema method
@@ -123,15 +124,11 @@ module Langchain::Vectorsearch
     # @param k [Integer] The number of top results to return
     # @return [Array<Hash>] The results of the search
     def similarity_search_by_vector(embedding:, k: 4)
-      result = client.transaction do |conn|
-        conn.exec("SET LOCAL ivfflat.probes = 10;")
-        query = <<~SQL
-          SELECT id, content FROM #{quoted_table_name} ORDER BY vectors #{operator} $1 ASC LIMIT $2;
-        SQL
-        conn.exec_params(query, [embedding, k])
+      db.transaction do # BEGIN
+        documents_model
+          .nearest_neighbors(:vectors, embedding, distance: operator).limit(k)
+          .where(namespace_column.to_sym => namespace)
       end
-
-      result.to_a
     end
 
     # Ask a question and return the answer
@@ -142,7 +139,7 @@ module Langchain::Vectorsearch
       search_results = similarity_search(query: question)
 
       context = search_results.map do |result|
-        result["content"].to_s
+        result.content.to_s
       end
       context = context.join("\n---\n")
 

data/lib/langchain/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Langchain
-  VERSION = "0.6.8"
+  VERSION = "0.6.9"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: langchainrb
 version: !ruby/object:Gem::Version
-  version: 0.6.8
+  version: 0.6.9
 platform: ruby
 authors:
 - Andrei Bondarev
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-07-21 00:00:00.000000000 Z
+date: 2023-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: baran
@@ -324,14 +324,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: 0.2.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: 0.2.1
 - !ruby/object:Gem::Dependency
   name: pdf-reader
   requirement: !ruby/object:Gem::Requirement