bx_builder_chain 0.1.0

data/lib/bx_builder_chain/utils/token_length/base_validator.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module BxBuilderChain
+  module Utils
+    module TokenLength
+      #
+      # Calculate the `max_tokens:` parameter to be set by calculating the context length of the text minus the prompt length
+      #
+      # @param content [String | Array<String>] The text or array of texts to validate
+      # @param model_name [String] The model name to validate against
+      # @return [Integer] Whether the text is valid or not
+      # @raise [TokenLimitExceeded] If the text is too long
+      #
+      class BaseValidator
+        def self.validate_max_tokens!(content, model_name, options = {})
+          text_token_length = if content.is_a?(Array)
+            content.sum { |item| token_length(item.to_json, model_name, options) }
+          else
+            token_length(content, model_name, options)
+          end
+
+          leftover_tokens = token_limit(model_name) - text_token_length
+
+          if leftover_tokens < 0
+            raise "This model's maximum context length is #{token_limit(model_name)} tokens, but the given text is #{text_token_length} tokens long."
+          end
+
+          leftover_tokens
+        end
+
+        def self.token_length(text, model_name = "", options = {})
+          start_time = Time.new.to_f * 1000
+          settings = BxBuilderChain::Utils::Tokenization::OpenAiEncodings.cl100k_base
+          puts "loading 100k_base = #{(Time.new.to_f * 1000) - start_time}"
+          encoder = BxBuilderChain::Utils::Tokenization::BytePairEncoding.new(
+            pat_str: settings["pat_str"],
+            mergeable_ranks: settings["mergeable_ranks"]
+          )
+
+          encoder.encode(text).count
+        end
+      end
+    end
+  end
+end

data/lib/bx_builder_chain/utils/token_length/open_ai_validator.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module BxBuilderChain
+  module Utils
+    module TokenLength
+      #
+      # This class is meant to validate the length of the text passed in to OpenAI's API.
+      # It is used to validate the token length before the API call is made
+      #
+      class OpenAiValidator < BaseValidator
+        TOKEN_LIMITS = {
+          # Source:
+          # https://platform.openai.com/docs/api-reference/embeddings
+          # https://platform.openai.com/docs/models/gpt-4
+          "text-embedding-ada-002" => 8191,
+          "gpt-3.5-turbo" => 4096,
+          "gpt-3.5-turbo-0301" => 4096,
+          "gpt-3.5-turbo-0613" => 4096,
+          "gpt-3.5-turbo-16k" => 16384,
+          "gpt-3.5-turbo-16k-0613" => 16384,
+          "text-davinci-003" => 4097,
+          "text-davinci-002" => 4097,
+          "code-davinci-002" => 8001,
+          "gpt-4" => 8192,
+          "gpt-4-0314" => 8192,
+          "gpt-4-0613" => 8192,
+          "gpt-4-32k" => 32768,
+          "gpt-4-32k-0314" => 32768,
+          "gpt-4-32k-0613" => 32768,
+          "text-curie-001" => 2049,
+          "text-babbage-001" => 2049,
+          "text-ada-001" => 2049,
+          "davinci" => 2049,
+          "curie" => 2049,
+          "babbage" => 2049,
+          "ada" => 2049
+        }.freeze
+
+        def self.token_limit(model_name)
+          TOKEN_LIMITS[model_name]
+        end
+                def self.validate_max_tokens!(content, model_name, options = {})
+          text_token_length = if content.is_a?(Array)
+            content.sum { |item| token_length(item.to_json, model_name, options) }
+          else
+            token_length(content, model_name, options)
+          end
+
+          leftover_tokens = token_limit(model_name) - text_token_length
+
+          if leftover_tokens < 0
+            raise "This model's maximum context length is #{token_limit(model_name)} tokens, but the given text is #{text_token_length} tokens long."
+          end
+
+          leftover_tokens
+        end
+
+        def self.token_length(text, model_name = "", options = {})
+          settings = BxBuilderChain::Utils::Tokenization::OpenAiEncodings.cl100k_base
+          encoder = BxBuilderChain::Utils::Tokenization::BytePairEncoding.new(
+            pat_str: settings["pat_str"],
+            mergeable_ranks: settings["mergeable_ranks"]
+          )
+          
+          encoder.encode(text).count
+        end
+      end
+    end
+  end
+end

data/lib/bx_builder_chain/utils/tokenization/byte_pair_encoding.rb

@@ -0,0 +1,72 @@
+module BxBuilderChain
+  module Utils
+    module Tokenization
+      class BytePairEncoding
+        def initialize(pat_str:, mergeable_ranks:)
+          @pat_str = pat_str
+          @mergeable_ranks = mergeable_ranks
+          @decoder = mergeable_ranks.invert
+          @pat = Regexp.new(pat_str)
+          @tokenized_string = []
+        end
+
+        def encode(text, visualise: nil)
+          words = text.scan(@pat)
+          words.flat_map { |word| bpe_encode(word.bytes, visualise: visualise) }
+        end
+
+        def visualise_tokenised_string
+          visualise_tokens_coloured(@tokenized_string) unless @tokenized_string.empty?
+        end
+
+        private
+
+        def bpe_encode(input, visualise: nil)
+          parts = input.map(&:chr)
+          visualiser = select_visualiser(visualise)
+
+          until (min_pair = find_min_rank_pair(parts)).nil?
+            visualiser.call(parts)
+            idx = min_pair[:index]
+            parts[idx, 2] = [parts[idx] + parts[idx + 1]]
+          end
+
+          @tokenized_string.concat(parts)
+          puts '' unless visualise.nil?
+          parts.map { |part| @mergeable_ranks[part] }
+        end
+
+        def find_min_rank_pair(parts)
+            pairs = parts.each_cons(2).with_index.map do |pair, i|
+              rank = @mergeable_ranks[pair.join]
+              { pair: pair, rank: rank, index: i } if rank
+            end.compact
+            pairs.min_by { |entry| entry[:rank] }
+        end
+
+        def select_visualiser(mode)
+          case mode
+          when 'colour', 'color'
+            method(:visualise_tokens_coloured)
+          when 'simple'
+            method(:visualise_tokens_simple)
+          else
+            ->(_tokens) {}
+          end
+        end
+
+        def visualise_tokens_coloured(token_values)
+          backgrounds = [167, 179, 185, 77, 80, 68, 134].cycle
+          output = token_values.each_with_index.map do |value, index|
+            "\e[48;5;#{backgrounds.next}m#{value}"
+          end.join
+          puts "#{output}\e[0m"
+        end
+
+        def visualise_tokens_simple(token_values)
+          puts token_values
+        end
+      end
+    end
+  end
+end

data/lib/bx_builder_chain/utils/tokenization/open_ai_encodings.rb

@@ -0,0 +1,44 @@
+require 'base64'
+
+module BxBuilderChain
+  module Utils
+    module Tokenization
+      class OpenAiEncodings
+
+
+        ENDOFTEXT = "<|endoftext|>"
+        FIM_PREFIX = "<|fim_prefix|>"
+        FIM_MIDDLE = "<|fim_middle|>"
+        FIM_SUFFIX = "<|fim_suffix|>"
+        ENDOFPROMPT = "<|endofprompt|>"
+
+        def self.load_tiktoken_bpe(tiktoken_bpe_file)
+          contents = File.read(tiktoken_bpe_file)
+          contents.split("\n").each_with_object({}) do |line, hash|
+            token, rank = line.split
+            hash[Base64.decode64(token)] = rank.to_i
+          end
+        end
+
+        def self.cl100k_base
+          mergeable_ranks = load_tiktoken_bpe(File.join(__dir__, '..', 'token_data', 'cl100k_base.tiktoken'))
+
+          special_tokens = {
+            ENDOFTEXT => 100257,
+            FIM_PREFIX => 100258,
+            FIM_MIDDLE => 100259,
+            FIM_SUFFIX => 100260,
+            ENDOFPROMPT => 100276,
+          }
+
+          {
+            "name" => "cl100k_base",
+            "pat_str" => /(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+/,
+            "mergeable_ranks" => mergeable_ranks,
+            "special_tokens" => special_tokens
+          }
+        end
+      end
+    end
+  end
+end

data/lib/bx_builder_chain/vectorsearch/base.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+module BxBuilderChain::Vectorsearch
+  # = Vector Databases
+  # A vector database a type of database that stores data as high-dimensional vectors, which are mathematical representations of features or attributes. Each vector has a certain number of dimensions, which can range from tens to thousands, depending on the complexity and granularity of the data.
+  #
+  # == Available vector databases
+  #
+  # - {BxBuilderChain::Vectorsearch::Pgvector}
+  #
+  # == Usage
+  #
+  # 1. Pick a vector database from list.
+  # 2. Review its documentation to install the required gems, and create an account, get an API key, etc
+  # 3. Instantiate the vector database class:
+  #
+  #     weaviate = BxBuilderChain::Vectorsearch::Weaviate.new(
+  #       url:         ENV["WEAVIATE_URL"],
+  #       api_key:     ENV["WEAVIATE_API_KEY"],
+  #       table_name:  "Documents",
+  #       llm:         :openai,              # or :cohere, :hugging_face, :google_palm, or :replicate
+  #       llm_api_key: ENV["OPENAI_API_KEY"] # API key for the selected LLM
+  #     )
+  #
+  #     # You can instantiate other supported vector databases the same way:
+  #     milvus   = BxBuilderChain::Vectorsearch::Milvus.new(...)
+  #     qdrant   = BxBuilderChain::Vectorsearch::Qdrant.new(...)
+  #     pinecone = BxBuilderChain::Vectorsearch::Pinecone.new(...)
+  #     chrome   = BxBuilderChain::Vectorsearch::Chroma.new(...)
+  #     pgvector = BxBuilderChain::Vectorsearch::Pgvector.new(...)
+  #
+  # == Schema Creation
+  #
+  # `create_default_schema()` creates default schema in your vector database.
+  #
+  #     search.create_default_schema
+  #
+  # (We plan on offering customizable schema creation shortly)
+  #
+  # == Adding Data
+  #
+  # You can add data with:
+  # 1. `add_data(path:, paths:)` to add any kind of data type
+  #
+  #     my_pdf = BxBuilderChain.root.join("path/to/my.pdf")
+  #     my_text = BxBuilderChain.root.join("path/to/my.txt")
+  #     my_docx = BxBuilderChain.root.join("path/to/my.docx")
+  #     my_csv = BxBuilderChain.root.join("path/to/my.csv")
+  #
+  #     search.add_data(paths: [my_pdf, my_text, my_docx, my_csv])
+  #
+  # 2. `add_texts(texts:)` to only add textual data
+  #
+  #     search.add_texts(
+  #       texts: [
+  #         "Lorem Ipsum is simply dummy text of the printing and typesetting industry.",
+  #         "Lorem Ipsum has been the industry's standard dummy text ever since the 1500s"
+  #       ]
+  #     )
+  #
+  # == Retrieving Data
+  #
+  # `similarity_search_by_vector(embedding:, k:)` searches the vector database for the closest `k` number of embeddings.
+  #
+  #    search.similarity_search_by_vector(
+  #      embedding: ...,
+  #      k: # number of results to be retrieved
+  #    )
+  #
+  # `vector_store.similarity_search(query:, k:)` generates an embedding for the query and searches the vector database for the closest `k` number of embeddings.
+  #
+  # search.similarity_search_by_vector(
+  #   embedding: ...,
+  #   k: # number of results to be retrieved
+  # )
+  #
+  # `ask(question:)` generates an embedding for the passed-in question, searches the vector database for closest embeddings and then passes these as context to the LLM to generate an answer to the question.
+  #
+  #     search.ask(question: "What is lorem ipsum?")
+  #
+  class Base
+    include BxBuilderChain::DependencyHelper
+
+    attr_reader :client, :table_name, :llm
+
+    DEFAULT_METRIC = "cosine"
+
+    # @param llm [Object] The LLM client to use
+    def initialize(llm:)
+      @llm = llm
+    end
+
+    # Method supported by Vectorsearch DB to retrieve a default schema
+    def get_default_schema
+      raise NotImplementedError, "#{self.class.name} does not support retrieving a default schema"
+    end
+
+    # Method supported by Vectorsearch DB to create a default schema
+    def create_default_schema
+      raise NotImplementedError, "#{self.class.name} does not support creating a default schema"
+    end
+
+    # Method supported by Vectorsearch DB to delete the default schema
+    def destroy_default_schema
+      raise NotImplementedError, "#{self.class.name} does not support deleting a default schema"
+    end
+
+    # Method supported by Vectorsearch DB to add a list of texts to the index
+    def add_texts(**kwargs)
+      raise NotImplementedError, "#{self.class.name} does not support adding texts"
+    end
+
+    # Method supported by Vectorsearch DB to update a list of texts to the index
+    def update_texts(**kwargs)
+      raise NotImplementedError, "#{self.class.name} does not support updating texts"
+    end
+
+    # Method supported by Vectorsearch DB to search for similar texts in the index
+    def similarity_search(**kwargs)
+      raise NotImplementedError, "#{self.class.name} does not support similarity search"
+    end
+
+    # Method supported by Vectorsearch DB to search for similar texts in the index by the passed in vector.
+    # You must generate your own vector using the same LLM that generated the embeddings stored in the Vectorsearch DB.
+    def similarity_search_by_vector(**kwargs)
+      raise NotImplementedError, "#{self.class.name} does not support similarity search by vector"
+    end
+
+    # Method supported by Vectorsearch DB to answer a question given a context (data) pulled from your Vectorsearch DB.
+    def ask(**kwargs)
+      raise NotImplementedError, "#{self.class.name} does not support asking questions"
+    end
+
+    def generate_prompt(question:, context:, prompt_template: nil)
+      template = prompt_template || BxBuilderChain.configuration.default_prompt_template
+      template % {context: context, question: question}
+    end
+
+    def add_data(paths:)
+      raise ArgumentError, "Paths must be provided" if Array(paths).empty?
+
+      texts = Array(paths)
+        .flatten
+        .map do |path|
+          data = BxBuilderChain::Loader.new(path)&.load&.chunks
+          data.map { |chunk| chunk[:text] }
+        end
+
+      texts.flatten!
+
+      add_texts(texts: texts)
+    end
+
+    def self.logger_options
+      {
+        color: :blue
+      }
+    end
+  end
+end

data/lib/bx_builder_chain/vectorsearch/pgvector.rb

@@ -0,0 +1,228 @@
+# frozen_string_literal: true
+
+module BxBuilderChain::Vectorsearch
+  class Pgvector < Base
+    #
+    # The PostgreSQL vector search adapter
+    #
+    # Usage:
+    # pgvector = Langchain::Vectorsearch::Pgvector.new(url:, table_name:, llm:, namespace: nil)
+    #
+
+    # The operators supported by the PostgreSQL vector search adapter
+    OPERATORS = {
+      "cosine_distance" => "cosine",
+      "euclidean_distance" => "euclidean"
+    }
+    DEFAULT_OPERATOR = "cosine_distance"
+
+    attr_reader :db, :operator, :table_name, :namespace_column, :namespaces, :documents_table
+
+    # @param url [String] The URL of the PostgreSQL database
+    # @param table_name [String] The name of the table to use for the index
+    # @param llm [Object] The LLM client to use
+    # @param namespace [String] The namespace to use for the index when inserting/querying
+    def initialize(llm:, namespaces: [BxBuilderChain.configuration.public_namespace] || ['public'])
+
+      depends_on "sequel"
+      require "sequel"
+
+      @db = Sequel.connect(BxBuilderChain.configuration.pg_url)
+      @table_name = "bx_builder_chain_embeddings"
+      @namespace_column = "namespace"
+      set_namespaces(namespaces)
+      @threshold = BxBuilderChain.configuration.threshold
+
+      validate_threshold(@threshold)
+
+      @operator = OPERATORS[DEFAULT_OPERATOR]
+
+      super(llm: llm)
+    end
+
+    def documents_model
+      Class.new(Sequel::Model(@table_name.to_sym)) do
+        plugin :pgvector, :vectors
+      end
+    end
+
+    # Upsert a list of texts to the index
+    # @param texts [Array<String>] The texts to add to the index
+    # @param ids [Array<Integer>] The ids of the objects to add to the index, in the same order as the texts
+    # @return [PG::Result] The response from the database including the ids of
+    # the added or updated texts.
+    def upsert_texts(texts:, ids:)
+      data = texts.zip(ids).flat_map do |(text, id)|
+        {id: id, content: text, vectors: llm.embed(text: text).to_s, namespace: namespaces[0]}
+      end
+      # @db[table_name.to_sym].multi_insert(data, return: :primary_key)
+      @db[@table_name.to_sym]
+        .insert_conflict(
+          target: :id,
+          update: {content: Sequel[:excluded][:content], vectors: Sequel[:excluded][:vectors]}
+        )
+        .multi_insert(data, return: :primary_key)
+    end
+
+    # Add a list of texts to the index
+    # @param texts [Array<String>] The texts to add to the index
+    # @param ids [Array<String>] The ids to add to the index, in the same order as the texts
+    # @return [Array<Integer>] The the ids of the added texts.
+    def add_texts(texts:, ids: nil)
+      if ids.nil? || ids.empty?
+        data = texts.map do |text|
+          {content: text, vectors: llm.embed(text: text).to_s, namespace: namespaces[0]}
+        end
+        
+        @db[@table_name.to_sym].multi_insert(data, return: :primary_key)
+      else
+        upsert_texts(texts: texts, ids: ids)
+      end
+    end
+
+    # Update a list of ids and corresponding texts to the index
+    # @param texts [Array<String>] The texts to add to the index
+    # @param ids [Array<String>] The ids to add to the index, in the same order as the texts
+    # @return [Array<Integer>] The ids of the updated texts.
+    def update_texts(texts:, ids:)
+      upsert_texts(texts: texts, ids: ids)
+    end
+
+    def create_default_schema
+      db.run "CREATE EXTENSION IF NOT EXISTS vector"
+      
+      namespace_column = @namespace_column
+      vector_dimension = llm.default_dimension || 1000
+      
+      # bx_builder_chain_embeddings table
+      db.create_table? :bx_builder_chain_embeddings do
+        primary_key :id
+        text :content
+        column :vectors, "vector(#{vector_dimension})"
+        text namespace_column.to_sym, default: 'public'
+        
+        index namespace_column.to_sym
+      end
+    
+      # bx_builder_chain_documents table
+      db.create_table? :bx_builder_chain_documents do
+        primary_key :id
+        text :name
+        text namespace_column.to_sym, default: 'public'
+        timestamp :created_at
+        timestamp :updated_at
+        
+        index [:name, namespace_column.to_sym], unique: true
+      end
+      
+      # bx_builder_chain_document_chunks table
+      db.create_table? :bx_builder_chain_document_chunks do
+        primary_key :id
+        foreign_key :document_id, :bx_builder_chain_documents, null: false, on_delete: :cascade
+        foreign_key :embedding_id, :bx_builder_chain_embeddings, null: false, on_delete: :cascade
+    
+        unique [:document_id, :embedding_id]
+      end
+    end
+    
+
+    # Destroy default schema
+    def destroy_default_schema
+      db.drop_table? :bx_builder_chain_document_chunks
+      db.drop_table? :bx_builder_chain_documents
+      db.drop_table? :bx_builder_chain_embeddings
+    end
+
+    # Search for similar texts in the index
+    # @param query [String] The text to search for
+    # @param k [Integer] The number of top results to return
+    # @return [Array<Hash>] The results of the search
+    def similarity_search(query:, k: 4)
+      embedding = llm.embed(text: query)
+
+      similarity_search_by_vector(
+        embedding: embedding,
+        k: k
+      )
+    end
+
+    # Search for similar texts in the index by the passed in vector.
+    # You must generate your own vector using the same LLM that generated the embeddings stored in the Vectorsearch DB.
+    # @param embedding [Array<Float>] The vector to search for
+    # @param k [Integer] The number of top results to return
+    # @return [Array<Hash>] The results of the search
+    def similarity_search_by_vector(embedding:, k: 4)
+      db.transaction do # BEGIN
+        documents_model
+          .nearest_neighbors(:vectors, embedding, distance: operator, threshold: @threshold)
+          .where(@namespace_column.to_sym => namespaces)
+          .limit(k)
+      end
+    end
+
+    # Ask a question and return the answer
+    # @param question [String] The question to ask
+    # @return [String] The answer to the question
+    def ask(question:, context_results: 4, prompt_template: nil)
+      search_results = similarity_search(query: question, k: context_results)
+
+      context = search_results.map do |result|
+        result.content.to_s
+      end
+      context = context.join("\n---\n")
+
+      prompt = generate_prompt(question: question, context: context, prompt_template: nil)
+
+      llm.chat(prompt: prompt)
+    end
+
+    def add_data(paths:)
+      raise ArgumentError, "Paths must be provided" if Array(paths).empty?
+
+      all_added_chunk_ids = []
+
+      @db.transaction do  # Start the transaction
+        paths.each do |file_n_path|
+          path, file = extract_path_and_file(file_n_path)
+
+          texts = BxBuilderChain::Loader.new(path)&.load&.chunks.map { |chunk| chunk[:text] }
+          
+          texts.flatten!
+
+          added_chunk_ids_for_current_path = add_texts(texts: texts)
+          
+          all_added_chunk_ids.concat(added_chunk_ids_for_current_path)
+    
+          document_record_id = @db[:bx_builder_chain_documents].insert(
+                                                                  name: file, 
+                                                                  namespace: namespaces[0],
+                                                                  created_at: Time.now.utc,
+                                                                  updated_at: Time.now.utc
+                                                                )
+    
+          document_chunks_data = added_chunk_ids_for_current_path.map do |chunk_id|
+            {document_id: document_record_id, embedding_id: chunk_id}
+          end
+          @db[:bx_builder_chain_document_chunks].multi_insert(document_chunks_data)
+        end
+      end  # End the transaction
+
+      all_added_chunk_ids
+    end
+
+    private 
+
+    def extract_path_and_file(entry)
+      entry.is_a?(Hash) ? [entry[:path], entry[:filename]] : [entry, entry]
+    end
+
+    def validate_threshold(threshold)
+      raise "Threshold must be between 0.0 and 2.0 (0.0 being a perfect match) or nil (ignore threshold)" if !threshold.nil? && (threshold > 2 || threshold < 0)
+    end
+
+    def set_namespaces(namespaces)
+      namespaces = [namespaces] unless namespaces.is_a?(Array)
+      @namespaces = namespaces.push(BxBuilderChain.configuration.public_namespace).uniq  
+    end    
+  end
+end

data/lib/bx_builder_chain/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module BxBuilderChain
+  VERSION = "0.1.0"
+end

data/lib/bx_builder_chain.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require "logger"
+require 'dotenv'
+require "zeitwerk"
+
+Dotenv.load
+loader = Zeitwerk::Loader.for_gem(warn_on_extra_files: false)
+loader.ignore("#{__dir__}/bx_builder_chain.rb")
+loader.setup
+
+module BxBuilderChain
+  class << self
+    attr_reader :logger
+    attr_writer :configuration
+
+    def logger=(logger)
+      @logger ||= logger
+    end
+    
+    def configuration
+      @configuration ||= Configuration.new
+    end
+  
+    def configure
+      yield(configuration)
+    end
+
+    def reset_config
+      @configuration = Configuration.new
+    end
+  end
+
+  self.logger = ::Logger.new($stdout, level: :warn)
+
+  class Error < StandardError; end
+
+end