langchainrb 0.7.0 → 0.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4efc896c1c0fa895ebd11bdb3c4d5604ccd34878aa29472419efca85800072da
-  data.tar.gz: c703f8150c7a6a6cb802260da2eeb95d4f7542cc0c5490794cea844351b4fe7c
+  metadata.gz: 21e6cb42af2a2a6892ab2c4dd76ad993b41574ca7a903702997ad20a9380ff6e
+  data.tar.gz: 620eb70528fb4bbeaf6c9b268717d491e4f74063ea4a897404d3ac429f9f1b93
 SHA512:
-  metadata.gz: 18b0c48b747978b2a92ecd8d118f31b199d5f58469a5dd55ccbfc3a56a9044faf42784e4760ff9a9fca94da019107629c107a3fe586b7d55243aa92bd1c5b949
-  data.tar.gz: d8513d2018ce48a60fbecc9ca3efb91411385fd71301e0c79ac3612d91b2b504427f2ea19b41991f2f4b43a8bd3b650035684b02a46ae0b114d83343ef5bce18
+  metadata.gz: 8a82bf546ca46559c966e0669266b6f9b6184f01268b5c82ebfa312a400f9b2480479550fdf78341ccbd05a9c170a44ae0730fb3b9ea594f6d8bd59484b7699b
+  data.tar.gz: cae88e17f88a407c16caa29b69b61fcede6e1655c05d1b1710496852c921e036bf1d732dc31d391b07deb402ec44098f36831ed7305e7789dff223b440db0438

data/CHANGELOG.md

@@ -1,5 +1,8 @@
 ## [Unreleased]
 
+## [0.7.1] - 2023-10-26
+- Ragas evals tool to evaluate Retrieval Augmented Generation (RAG) pipelines
+
 ## [0.7.0] - 2023-10-22
 - BREAKING: Moving Rails-specific code to `langchainrb_rails` gem
 

data/README.md

@@ -4,8 +4,6 @@
 
 👨‍💻👩‍💻 CURRENTLY SEEKING PEOPLE TO FORM THE CORE GROUP OF MAINTAINERS WITH
 
-:warning: UNDER ACTIVE AND RAPID DEVELOPMENT (MAY BE BUGGY AND UNTESTED)
-
 ![Tests status](https://github.com/andreibondarev/langchainrb/actions/workflows/ci.yml/badge.svg?branch=main)
 [![Gem Version](https://badge.fury.io/rb/langchainrb.svg)](https://badge.fury.io/rb/langchainrb)
 [![Docs](http://img.shields.io/badge/yard-docs-blue.svg)](http://rubydoc.info/gems/langchainrb)
@@ -521,6 +519,32 @@ Langchain::Loader.load('https://www.example.com/file.pdf')
 ## Examples
 Additional examples available: [/examples](https://github.com/andreibondarev/langchainrb/tree/main/examples)
 
+## Evaluations (Evals)
+The Evaluations module is a collection of tools that can be used to evaluate and track the performance of the output products by LLM and your RAG (Retrieval Augmented Generation) pipelines.
+
+### RAGAS
+Ragas is helps you evaluate your Retrieval Augmented Generation (RAG) pipelines. The implementation is based on this [paper](https://arxiv.org/abs/2309.15217) and the original Python [repo](https://github.com/explodinggradients/ragas). Ragas tracks the 3 following metrics and assigns the 0.0 - 1.0 scores:
+* Faithfulness - the answer is grounded in the given context
+* Context Relevance - the retrieved context is focused, containing as little irrelevant information as possible
+* Answer Relevance - the generated answer addresses the actual question that was provided
+
+```ruby
+# We recommend using Langchain::LLM::OpenAI as your llm for Ragas
+ragas = Langchain::Evals::Ragas::Main.new(llm: llm) 
+
+# The answer that the LLM generated
+# The question (or the original prompt) that was asked
+# The context that was retrieved (usually from a vectorsearch database)
+ragas.score(answer: "", question: "", context: "")
+# =>
+# {
+#   ragas_score: 0.6601257446503674,
+#   answer_relevance_score: 0.9573145866787608,
+#   context_relevance_score: 0.6666666666666666,
+#   faithfulness_score: 0.5
+# }
+```
+
 ## Logging
 
 LangChain.rb uses standard logging mechanisms and defaults to `:warn` level. Most messages are at info level, but we will add debug or warn statements as needed.

data/lib/langchain/evals/ragas/answer_relevance.rb

@@ -0,0 +1,71 @@
+# freeze_string_literal: true
+
+require "matrix"
+
+module Langchain
+  module Evals
+    module Ragas
+      # Answer Relevance refers to the idea that the generated answer should address the actual question that was provided.
+      # This metric evaluates how closely the generated answer aligns with the initial question or instruction.
+      class AnswerRelevance
+        attr_reader :llm, :batch_size
+
+        # @param llm [Langchain::LLM::*] Langchain::LLM::* object
+        # @param batch_size [Integer] Batch size, i.e., number of generated questions to compare to the original question
+        def initialize(llm:, batch_size: 3)
+          @llm = llm
+          @batch_size = batch_size
+        end
+
+        # @param question [String] Question
+        # @param answer [String] Answer
+        # @return [Float] Answer Relevance score
+        def score(question:, answer:)
+          generated_questions = []
+
+          batch_size.times do |i|
+            prompt = answer_relevance_prompt_template.format(
+              question: question,
+              answer: answer
+            )
+            generated_questions << llm.complete(prompt: prompt).completion
+          end
+
+          scores = generated_questions.map do |generated_question|
+            calculate_similarity(original_question: question, generated_question: generated_question)
+          end
+
+          # Find the mean
+          scores.sum(0.0) / scores.size
+        end
+
+        private
+
+        # @param question_1 [String] Question 1
+        # @param question_2 [String] Question 2
+        # @return [Float] Dot product similarity between the two questions
+        def calculate_similarity(original_question:, generated_question:)
+          original_embedding = generate_embedding(original_question)
+          generated_embedding = generate_embedding(generated_question)
+
+          vector_1 = Vector.elements(original_embedding)
+          vector_2 = Vector.elements(generated_embedding)
+          vector_1.inner_product(vector_2)
+        end
+
+        # @param text [String] Text to generate an embedding for
+        # @return [Array<Float>] Embedding
+        def generate_embedding(text)
+          llm.embed(text: text).embedding
+        end
+
+        # @return [PromptTemplate] PromptTemplate instance
+        def answer_relevance_prompt_template
+          @template ||= Langchain::Prompt.load_from_path(
+            file_path: Langchain.root.join("langchain/evals/ragas/prompts/answer_relevance.yml")
+          )
+        end
+      end
+    end
+  end
+end

data/lib/langchain/evals/ragas/context_relevance.rb

@@ -0,0 +1,46 @@
+# freeze_string_literal: true
+
+require "pragmatic_segmenter"
+
+module Langchain
+  module Evals
+    module Ragas
+      # Context Relevance refers to the idea that the retrieved context should be focused, containing as little irrelevant information as possible.
+      class ContextRelevance
+        attr_reader :llm
+
+        # @param llm [Langchain::LLM::*] Langchain::LLM::* object
+        def initialize(llm:)
+          @llm = llm
+        end
+
+        # @param question [String] Question
+        # @param context [String] Context
+        # @return [Float] Context Relevance score
+        def score(question:, context:)
+          prompt = context_relevance_prompt_template.format(
+            question: question,
+            context: context
+          )
+          sentences = llm.complete(prompt: prompt).completion
+
+          (sentence_count(sentences).to_f / sentence_count(context).to_f)
+        end
+
+        private
+
+        def sentence_count(context)
+          ps = PragmaticSegmenter::Segmenter.new(text: context)
+          ps.segment.length
+        end
+
+        # @return [PromptTemplate] PromptTemplate instance
+        def context_relevance_prompt_template
+          @template ||= Langchain::Prompt.load_from_path(
+            file_path: Langchain.root.join("langchain/evals/ragas/prompts/context_relevance.yml")
+          )
+        end
+      end
+    end
+  end
+end

data/lib/langchain/evals/ragas/faithfulness.rb

@@ -0,0 +1,83 @@
+# freeze_string_literal: true
+
+module Langchain
+  module Evals
+    module Ragas
+      # Faithfulness refers to the idea that the answer should be grounded in the given context,
+      # ensuring that the retrieved context can act as a justification for the generated answer.
+      # The answer is faithful to the context if the claims that are made in the answer can be inferred from the context.
+      #
+      # Score calculation:
+      # F = |V| / |S|
+      #
+      # F = Faithfulness
+      # |V| = Number of statements that were supported according to the LLM
+      # |S| = Total number of statements extracted.
+      #
+      class Faithfulness
+        attr_reader :llm
+
+        # @param llm [Langchain::LLM::*] Langchain::LLM::* object
+        def initialize(llm:)
+          @llm = llm
+        end
+
+        # @param question [String] Question
+        # @param answer [String] Answer
+        # @param context [String] Context
+        # @return [Float] Faithfulness score
+        def score(question:, answer:, context:)
+          statements = statements_extraction(question: question, answer: answer)
+          statements_count = statements
+            .split("\n")
+            .count
+
+          verifications = statements_verification(statements: statements, context: context)
+          verifications_count = count_verified_statements(verifications)
+
+          (verifications_count.to_f / statements_count.to_f)
+        end
+
+        private
+
+        def count_verified_statements(verifications)
+          match = verifications.match(/Final verdict for each statement in order:\s*(.*)/)
+          verdicts = match.captures.first
+          verdicts
+            .split(".")
+            .count { |value| value.strip.to_boolean }
+        end
+
+        def statements_verification(statements:, context:)
+          prompt = statements_verification_prompt_template.format(
+            statements: statements,
+            context: context
+          )
+          llm.complete(prompt: prompt).completion
+        end
+
+        def statements_extraction(question:, answer:)
+          prompt = statements_extraction_prompt_template.format(
+            question: question,
+            answer: answer
+          )
+          llm.complete(prompt: prompt).completion
+        end
+
+        # @return [PromptTemplate] PromptTemplate instance
+        def statements_verification_prompt_template
+          @template_two ||= Langchain::Prompt.load_from_path(
+            file_path: Langchain.root.join("langchain/evals/ragas/prompts/faithfulness_statements_verification.yml")
+          )
+        end
+
+        # @return [PromptTemplate] PromptTemplate instance
+        def statements_extraction_prompt_template
+          @template_one ||= Langchain::Prompt.load_from_path(
+            file_path: Langchain.root.join("langchain/evals/ragas/prompts/faithfulness_statements_extraction.yml")
+          )
+        end
+      end
+    end
+  end
+end

data/lib/langchain/evals/ragas/main.rb

@@ -0,0 +1,70 @@
+# freeze_string_literal: true
+
+module Langchain
+  module Evals
+    # The RAGAS (Retrieval Augmented Generative Assessment) is a framework for evaluating RAG (Retrieval Augmented Generation) pipelines.
+    # Based on the following research: https://arxiv.org/pdf/2309.15217.pdf
+    module Ragas
+      class Main
+        attr_reader :llm
+
+        def initialize(llm:)
+          @llm = llm
+        end
+
+        # Returns the RAGAS scores, e.g.:
+        # {
+        #   ragas_score: 0.6601257446503674,
+        #   answer_relevance_score: 0.9573145866787608,
+        #   context_relevance_score: 0.6666666666666666,
+        #   faithfulness_score: 0.5
+        # }
+        #
+        # @param question [String] Question
+        # @param answer [String] Answer
+        # @param context [String] Context
+        # @return [Hash] RAGAS scores
+        def score(question:, answer:, context:)
+          answer_relevance_score = answer_relevance.score(question: question, answer: answer)
+          context_relevance_score = context_relevance.score(question: question, context: context)
+          faithfulness_score = faithfulness.score(question: question, answer: answer, context: context)
+
+          {
+            ragas_score: ragas_score(answer_relevance_score, context_relevance_score, faithfulness_score),
+            answer_relevance_score: answer_relevance_score,
+            context_relevance_score: context_relevance_score,
+            faithfulness_score: faithfulness_score
+          }
+        end
+
+        private
+
+        # Overall RAGAS score (harmonic mean): https://github.com/explodinggradients/ragas/blob/1dd363e3e54744e67b0be85962a0258d8121500a/src/ragas/evaluation.py#L140-L143
+        #
+        # @param answer_relevance_score [Float] Answer Relevance score
+        # @param context_relevance_score [Float] Context Relevance score
+        # @param faithfulness_score [Float] Faithfulness score
+        # @return [Float] RAGAS score
+        def ragas_score(answer_relevance_score, context_relevance_score, faithfulness_score)
+          reciprocal_sum = (1.0 / answer_relevance_score) + (1.0 / context_relevance_score) + (1.0 / faithfulness_score)
+          (3 / reciprocal_sum)
+        end
+
+        # @return [Langchain::Evals::Ragas::AnswerRelevance] Class instance
+        def answer_relevance
+          @answer_relevance ||= Langchain::Evals::Ragas::AnswerRelevance.new(llm: llm)
+        end
+
+        # @return [Langchain::Evals::Ragas::ContextRelevance] Class instance
+        def context_relevance
+          @context_relevance ||= Langchain::Evals::Ragas::ContextRelevance.new(llm: llm)
+        end
+
+        # @return [Langchain::Evals::Ragas::Faithfulness] Class instance
+        def faithfulness
+          @faithfulness ||= Langchain::Evals::Ragas::Faithfulness.new(llm: llm)
+        end
+      end
+    end
+  end
+end

data/lib/langchain/evals/ragas/prompts/answer_relevance.yml

@@ -0,0 +1,10 @@
+_type: prompt
+input_variables:
+  - answer
+template: |
+  Generate question for the given answer.
+  Answer: The PSLV-C56 mission is scheduled to be launched on Sunday, 30 July 2023 at 06:30 IST / 01:00 UTC. It will be launched from the Satish Dhawan Space Centre, Sriharikota, Andhra Pradesh, India 
+  Question: When is the scheduled launch date and time for the PSLV-C56 mission, and where will it be launched from?
+
+  Answer: {answer}
+  Question:

data/lib/langchain/evals/ragas/prompts/context_relevance.yml

@@ -0,0 +1,10 @@
+_type: prompt
+input_variables:
+  - question
+  - context
+template: |
+  Please extract relevant sentences from the provided context that is absolutely required answer the following question. If no relevant sentences are found, or if you believe the question cannot be answered from the given context, return the phrase "Insufficient Information".  While extracting candidate sentences you're not allowed to make any changes to sentences from given context.
+
+  question:{question}
+  context:\n{context}
+  candidate sentences:\n

data/lib/langchain/evals/ragas/prompts/faithfulness_statements_extraction.yml

@@ -0,0 +1,9 @@
+_type: prompt
+input_variables:
+  - question
+  - answer
+template: |
+  Given a question and answer, create one or more statements from each sentence in the given answer.
+  question: {question}
+  answer: {answer}
+  statements:\n

data/lib/langchain/evals/ragas/prompts/faithfulness_statements_verification.yml

@@ -0,0 +1,27 @@
+_type: prompt
+input_variables:
+  - statements
+  - context
+template: |
+  Consider the given context and following statements, then determine whether they are supported by the information present in the context.
+  Provide a brief explanation for each statement before arriving at the verdict (Yes/No). Provide a final verdict for each statement in order at the end in the given format.
+  Do not deviate from the specified format.
+
+  Context:\nJohn is a student at XYZ University. He is pursuing a degree in Computer Science. He is enrolled in several courses this semester, including Data Structures, Algorithms, and Database Management. John is a diligent student and spends a significant amount of time studying and completing assignments. He often stays late in the library to work on his projects.
+  statements:\n1. John is majoring in Biology.\n2. John is taking a course on Artificial Intelligence.\n3. John is a dedicated student.\n4. John has a part-time job.\n5. John is interested in computer programming.\n
+  Answer:
+  1. John is majoring in Biology.
+  Explanation: John's major is explicitly mentioned as Computer Science. There is no information suggesting he is majoring in Biology.  Verdict: No.
+  2. John is taking a course on Artificial Intelligence.
+  Explanation: The context mentions the courses John is currently enrolled in, and Artificial Intelligence is not mentioned. Therefore, it cannot be deduced that John is taking a course on AI. Verdict: No.
+  3. John is a dedicated student.
+  Explanation: The prompt states that he spends a significant amount of time studying and completing assignments. Additionally, it mentions that he often stays late in the library to work on his projects, which implies dedication. Verdict: Yes.
+  4. John has a part-time job.
+  Explanation: There is no information given in the context about John having a part-time job. Therefore, it cannot be deduced that John has a part-time job.  Verdict: No.
+  5. John is interested in computer programming.
+  Explanation: The context states that John is pursuing a degree in Computer Science, which implies an interest in computer programming. Verdict: Yes.
+  Final verdict for each statement in order: No. No. Yes. No. Yes.
+
+  context:\n{context}
+  statements:\n{statements}
+  Answer:

data/lib/langchain/utils/cosine_similarity.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Langchain
+  module Utils
+    class CosineSimilarity
+      attr_reader :vector_a, :vector_b
+
+      # @param vector_a [Array<Float>] First vector
+      # @param vector_b [Array<Float>] Second vector
+      def initialize(vector_a, vector_b)
+        @vector_a = vector_a
+        @vector_b = vector_b
+      end
+
+      # Calculate the cosine similarity between two vectors
+      # @return [Float] The cosine similarity between the two vectors
+      def calculate_similarity
+        return nil unless vector_a.is_a? Array
+        return nil unless vector_b.is_a? Array
+        return nil if vector_a.size != vector_b.size
+
+        dot_product = 0
+        vector_a.zip(vector_b).each do |v1i, v2i|
+          dot_product += v1i * v2i
+        end
+
+        a = vector_a.map { |n| n**2 }.reduce(:+)
+        b = vector_b.map { |n| n**2 }.reduce(:+)
+
+        dot_product / (Math.sqrt(a) * Math.sqrt(b))
+      end
+    end
+  end
+end

data/lib/langchain/vectorsearch/base.rb

@@ -25,8 +25,7 @@ module Langchain::Vectorsearch
   #       url:         ENV["WEAVIATE_URL"],
   #       api_key:     ENV["WEAVIATE_API_KEY"],
   #       index_name:  "Documents",
-  #       llm:         :openai,              # or :cohere, :hugging_face, :google_palm, or :replicate
-  #       llm_api_key: ENV["OPENAI_API_KEY"] # API key for the selected LLM
+  #       llm:         Langchain::LLM::OpenAI.new(api_key:)
   #     )
   #
   #     # You can instantiate other supported vector databases the same way:

data/lib/langchain/vectorsearch/chroma.rb

@@ -9,7 +9,7 @@ module Langchain::Vectorsearch
     #     gem "chroma-db", "~> 0.6.0"
     #
     # Usage:
-    # chroma = Langchain::Vectorsearch::Chroma.new(url:, index_name:, llm:, llm_api_key:, api_key: nil)
+    # chroma = Langchain::Vectorsearch::Chroma.new(url:, index_name:, llm:, api_key: nil)
     #
 
     # Initialize the Chroma client

data/lib/langchain/vectorsearch/pinecone.rb

@@ -9,7 +9,7 @@ module Langchain::Vectorsearch
     #     gem "pinecone", "~> 0.1.6"
     #
     # Usage:
-    #     pinecone = Langchain::Vectorsearch::Pinecone.new(environment:, api_key:, index_name:, llm:, llm_api_key:)
+    #     pinecone = Langchain::Vectorsearch::Pinecone.new(environment:, api_key:, index_name:, llm:)
     #
 
     # Initialize the Pinecone client

data/lib/langchain/vectorsearch/qdrant.rb

@@ -9,7 +9,7 @@ module Langchain::Vectorsearch
     #     gem "qdrant-ruby", "~> 0.9.3"
     #
     # Usage:
-    #     qdrant = Langchain::Vectorsearch::Qdrant.new(url:, api_key:, index_name:, llm:, llm_api_key:)
+    #     qdrant = Langchain::Vectorsearch::Qdrant.new(url:, api_key:, index_name:, llm:)
     #
 
     # Initialize the Qdrant client

data/lib/langchain/vectorsearch/weaviate.rb

@@ -9,7 +9,7 @@ module Langchain::Vectorsearch
     #     gem "weaviate-ruby", "~> 0.8.3"
     #
     # Usage:
-    #     weaviate = Langchain::Vectorsearch::Weaviate.new(url:, api_key:, index_name:, llm:, llm_api_key:)
+    #     weaviate = Langchain::Vectorsearch::Weaviate.new(url:, api_key:, index_name:, llm:)
     #
 
     # Initialize the Weaviate adapter

data/lib/langchain/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Langchain
-  VERSION = "0.7.0"
+  VERSION = "0.7.1"
 end

data/lib/langchain.rb

@@ -3,6 +3,7 @@
 require "logger"
 require "pathname"
 require "colorize"
+require "to_bool"
 require "zeitwerk"
 loader = Zeitwerk::Loader.for_gem
 loader.ignore("#{__dir__}/langchainrb.rb")

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: langchainrb
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.7.1
 platform: ruby
 authors:
 - Andrei Bondarev
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-10-23 00:00:00.000000000 Z
+date: 2023-10-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: baran
@@ -94,6 +94,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.3.0
+- !ruby/object:Gem::Dependency
+  name: to_bool
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+- !ruby/object:Gem::Dependency
+  name: matrix
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: dotenv-rails
   requirement: !ruby/object:Gem::Requirement
@@ -561,6 +589,14 @@ files:
 - lib/langchain/conversation/response.rb
 - lib/langchain/data.rb
 - lib/langchain/dependency_helper.rb
+- lib/langchain/evals/ragas/answer_relevance.rb
+- lib/langchain/evals/ragas/context_relevance.rb
+- lib/langchain/evals/ragas/faithfulness.rb
+- lib/langchain/evals/ragas/main.rb
+- lib/langchain/evals/ragas/prompts/answer_relevance.yml
+- lib/langchain/evals/ragas/prompts/context_relevance.yml
+- lib/langchain/evals/ragas/prompts/faithfulness_statements_extraction.yml
+- lib/langchain/evals/ragas/prompts/faithfulness_statements_verification.yml
 - lib/langchain/llm/ai21.rb
 - lib/langchain/llm/anthropic.rb
 - lib/langchain/llm/base.rb
@@ -607,6 +643,7 @@ files:
 - lib/langchain/tool/ruby_code_interpreter.rb
 - lib/langchain/tool/weather.rb
 - lib/langchain/tool/wikipedia.rb
+- lib/langchain/utils/cosine_similarity.rb
 - lib/langchain/utils/token_length/ai21_validator.rb
 - lib/langchain/utils/token_length/base_validator.rb
 - lib/langchain/utils/token_length/cohere_validator.rb