mistral_rb 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2e43cdb7fa62132f8342d13ec666a4fe48571dc0dbf5b64a2476c23d4da86aa5
-  data.tar.gz: 402b62dfe6f4ca6a2b77d3e13127b673037374fe75ffbe874c4cafef2fbdaeb3
+  metadata.gz: 9a420d04befbc7d822678ac18d5d251e9a3804a4f50413efe6743c05f8781c61
+  data.tar.gz: 572d1b152567b1ac54cd59ab0137eaef286732e288afcbfd79b025ced473b10d
 SHA512:
-  metadata.gz: 921683a278ab7f4d9f443c2db7a0dd16dd08b81752e6d99ac49fe619fb23e7d2cd145d30e8760988786ffa81647f9d666a73d0ac3a56e6c34ce921bd61a8339e
-  data.tar.gz: cbc90300023388689b9b5572f25dec507de1cc956f90c7406cfe8fbe685011200a248bbbb7574e564b3ff0b89a9726c60890c9175ac8e4832018b4766d1dabaf
+  metadata.gz: ca18ed1fb53190fe6146c26992e2e84525dcc596189396f883871c85dfb55507bb25c399cd290bcdadf72bd781740b8429533b3a6d05249f9a2ef0fff84551b8
+  data.tar.gz: 4544e24b6fb292b3b489e5969b3334e75ef035c340d8d7118275c43ffc2ca23413feab2cc881d1310c203c567d0f9d84ceb2eadbc2035223e29963f78d99fa75

data/README.md

@@ -44,7 +44,7 @@ end
 Here is how to use streaming:
 
 ```ruby
-api = MistralAPI.new("api_key")
+api = MistralAPI.new(api_key: "api_key")
 
 api.create_chat_completion(
   model: 'mistral-tiny',
@@ -56,6 +56,21 @@ end
 
 ```
 
+It is now possible add RAG (Retrieval Augmented Generation) in your apps, with only 6 lines of code:
+
+```ruby
+
+vector_store = PineconeService.new(index_name: 'your_index_name')
+llm = MistralAPI.new
+file = "https://www.ycombinator.com/deal"
+embedding_creator = MistralEmbeddingCreator.new
+
+responder = Responder.new(vector_store: vector_store,llm: llm,file: file, embedding_creator: embedding_creator)
+
+puts responder.call("How much does YC invest per startup ?")
+
+```
+
 
 ## Development
 

data/lib/content_extractors/content_extractor_factory.rb

@@ -0,0 +1,58 @@
+require "mime/types"
+require "httparty"
+require_relative './docx_content_extractor.rb'
+require_relative './html_content_extractor.rb'
+require_relative './pdf_content_extractor.rb'
+require_relative './text_content_extractor.rb'
+
+class ContentExtractorFactory
+  def self.for(file)
+    type = determine_file_type(file)
+
+    case type
+    when :pdf
+      PdfContentExtractor.new(file)
+    when :docx
+      DocxContentExtractor.new(file)
+    when :txt
+      TxtContentExtractor.new(file)
+    when :html
+      HtmlContentExtractor.new(file)
+    else
+      raise "Unsupported file type: #{type}"
+    end
+  end
+
+  private
+
+  def self.determine_file_type(file)
+    if file_is_url?(file)
+      content_type = fetch_url_content_type_with_httparty(file)
+      return :html if content_type.include?('text/html')
+    else
+      content_type = file.content_type
+    end
+
+    case content_type
+    when 'application/pdf'
+      :pdf
+    when 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
+      :docx
+    when 'text/plain'
+      :txt
+    else
+      :unknown
+    end
+  end
+
+  def self.file_is_url?(file)
+    file.respond_to?(:to_str) && file.to_str =~ /\A#{URI::regexp(['http', 'https'])}\z/
+  end
+
+  def self.fetch_url_content_type_with_httparty(url)
+    response = HTTParty.head(url)
+    response.headers['content-type']
+  rescue HTTParty::Error
+    :unknown
+  end
+end

data/lib/content_extractors/docx_content_extractor.rb

@@ -0,0 +1,77 @@
+require 'docx'
+require 'tempfile'
+require_relative '../utils/sanitizer.rb'
+require_relative '../utils/adapters.rb'
+
+class DocxContentExtractor
+  attr_reader :page_count
+
+  WORDS_PER_PAGE = 500
+
+  # Define custom error classes
+  class ExtractionError < StandardError; end
+  class FileDownloadError < ExtractionError; end
+  class FileReadError < ExtractionError; end
+  class DocxProcessingError < ExtractionError; end
+
+  def initialize(file)
+    @file = file
+  end
+
+  def call
+    extract_content
+  rescue StandardError => e
+    raise ExtractionError, "Content extraction failed: #{e.message}"
+  end
+
+  private
+
+  def extract_content
+    Tempfile.open(['temp', '.docx'], binmode: true) do |tempfile|
+      begin
+        @file.download { |chunk| tempfile.write(chunk.force_encoding("ASCII-8BIT")) }
+      rescue => e
+        raise FileDownloadError, "Failed to download file: #{e.message}"
+      end
+
+      begin
+        doc = Docx::Document.open(tempfile.path)
+      rescue => e
+        raise DocxProcessingError, "Failed to process DOCX file: #{e.message}"
+      end
+
+      content = extract_and_sanitize_content(doc)
+      pages = split_into_pages(content)
+      @page_count = pages.size
+
+      [pages, content]
+    end
+  end
+
+  def extract_and_sanitize_content(doc)
+    begin
+      content = doc.paragraphs.map(&:text).join("\n")
+      sanitize_page_content(content)
+    rescue => e
+      raise FileReadError, "Failed to read content from DOCX file: #{e.message}"
+    end
+  end
+
+  def split_into_pages(content)
+    words = content.split(/\s+/)
+    pages = []
+    words.each_slice(WORDS_PER_PAGE) do |page_words|
+      pages << page_words.join(' ')
+    end
+    pages
+  end
+
+  # Sanitize the content
+  def sanitize_page_content(content)
+    sanitized_text = Sanitizer.remove_excessive_newlines(content)
+    sanitized_text = Sanitizer.remove_excessive_spaces(sanitized_text)
+    sanitized_text = Sanitizer.remove_bullet_points(sanitized_text)
+    # Add additional sanitization methods as required
+    sanitized_text
+  end
+end

data/lib/content_extractors/html_content_extractor.rb

@@ -0,0 +1,83 @@
+require 'nokogiri'
+require 'watir'
+require 'webdrivers'
+require_relative '../utils/sanitizer.rb'
+
+class HtmlContentExtractor
+  attr_reader :content
+
+  WORDS_PER_PAGE = 500
+
+  class ExtractionError < StandardError; end
+  class UrlDownloadError < ExtractionError; end
+  class ParsingError < ExtractionError; end
+
+  BROWSERS = [:chrome, :firefox, :safari]
+
+  def initialize(url)
+    @url = url
+  end
+
+  def call
+    extract_content
+  rescue StandardError => e
+    raise ExtractionError, "HTML content extraction failed: #{e.message}"
+  end
+
+  private
+
+  def extract_content
+    BROWSERS.each do |browser|
+      begin
+        html = download_html_with_watir(browser)
+        document = parse_html(html)
+        text_content = document.xpath('//body').text.strip
+        title = document.title
+
+        combined_content = "#{title}\n\n#{text_content}"
+        @content = sanitize_content(combined_content)
+        pages = split_into_pages(@content)
+        return [pages, @content]
+      rescue UrlDownloadError => e
+        next
+      end
+    end
+
+    raise UrlDownloadError, "Failed to download URL with all browser drivers"
+  end
+
+  def download_html_with_watir(browser)
+    browser = Watir::Browser.new(browser, headless: true)
+    browser.goto(@url)
+    sleep(5)  # Adjust sleep time as needed for JavaScript to render
+    html_content = browser.html
+    browser.quit
+    html_content
+  rescue => e
+    raise UrlDownloadError, "Failed to download URL using #{browser.to_s.capitalize} browser: #{e.message}"
+  ensure
+    browser.quit if browser.exists?
+  end
+
+  def parse_html(html)
+    Nokogiri::HTML(html)
+  rescue => e
+    raise ParsingError, "Failed to parse HTML content: #{e.message}"
+  end
+
+  def sanitize_content(content)
+    sanitized_text = Sanitizer.remove_excessive_newlines(content)
+    sanitized_text = Sanitizer.remove_excessive_spaces(sanitized_text)
+    sanitized_text = Sanitizer.remove_bullet_points(sanitized_text)
+    sanitized_text
+  end
+
+  def split_into_pages(content)
+    words = content.split(/\s+/)
+    pages = []
+    words.each_slice(WORDS_PER_PAGE) do |page_words|
+      pages << page_words.join(' ')
+    end
+    pages
+  end
+end

data/lib/content_extractors/pdf_content_extractor.rb

@@ -0,0 +1,66 @@
+require 'pdf-reader'
+require 'tempfile'
+require_relative '../utils/sanitizer.rb'
+require_relative '../utils/adapters.rb'
+
+class PdfContentExtractor
+  attr_reader :page_count
+
+  # Define custom error classes
+  class ExtractionError < StandardError; end
+  class UnreadableContentError < ExtractionError; end
+  class EmptyContentError < ExtractionError; end
+
+  def initialize(file)
+    @file = file
+    @page_count = 0
+  end
+
+  def call
+    extract_content
+  rescue PDF::Reader::MalformedPDFError, PDF::Reader::UnsupportedFeatureError => e
+    # Handle known PDF::Reader errors
+    raise UnreadableContentError, "PDF could not be read: #{e.message}"
+  rescue StandardError => e
+    # Handle any other unforeseen errors
+    raise ExtractionError, "Content extraction failed: #{e.message}"
+  end
+
+  private
+
+  def extract_content
+    Tempfile.open(['extracted_content', '.pdf'], binmode: true) do |tempfile|
+      begin
+        @file.download { |chunk| tempfile.write(chunk.force_encoding("ASCII-8BIT")) }
+        tempfile.close # Close the tempfile to flush and save data before reading
+
+        reader = PDF::Reader.new(tempfile.path)
+        @page_count = reader.page_count # Store the page count
+
+        pages = reader.pages.map do |page|
+          # Encode the extracted text to UTF-8, replacing invalid characters
+          page_text = page.text.encode('UTF-8', invalid: :replace, undef: :replace, replace: '?')
+
+          # Sanitize the page text
+          sanitize_page_content(page_text)
+        end
+
+        raise EmptyContentError, 'The PDF content is empty or unreadable.' if pages.all? { |page| page.nil? || page.strip.empty? }
+
+        [pages, content]
+      ensure
+        tempfile.unlink # Delete the tempfile
+      end
+    end
+  end
+
+  # Sanitize the page content
+  def sanitize_page_content(page_text)
+    sanitized_text = Sanitizer.remove_excessive_newlines(page_text)
+    sanitized_text = Sanitizer.remove_excessive_spaces(sanitized_text)
+    sanitized_text = Sanitizer.remove_bullet_points(sanitized_text)
+    # Add additional sanitization methods as required
+
+    sanitized_text
+  end
+end

data/lib/content_extractors/text_content_extractor.rb

@@ -0,0 +1,66 @@
+require 'tempfile'
+require_relative '../utils/sanitizer.rb'
+require_relative '../utils/adapters.rb'
+
+class TxtContentExtractor
+  attr_reader :page_count
+
+  WORDS_PER_PAGE = 500
+
+  # Define custom error classes
+  class ExtractionError < StandardError; end
+  class FileDownloadError < ExtractionError; end
+  class FileReadError < ExtractionError; end
+
+  def initialize(file)
+    @file = file
+  end
+
+  def call
+    extract_content
+  rescue StandardError => e
+    raise ExtractionError, "Content extraction failed: #{e.message}"
+  end
+
+  private
+
+  def extract_content
+    Tempfile.open(['temp', '.txt'], binmode: true) do |tempfile|
+      begin
+        @file.download { |chunk| tempfile.write(chunk.force_encoding("UTF-8")) }
+      rescue => e
+        raise FileDownloadError, "Failed to download file: #{e.message}"
+      end
+
+      begin
+        content = File.read(tempfile.path)
+      rescue => e
+        raise FileReadError, "Failed to read file: #{e.message}"
+      end
+
+      sanitized_content = sanitize_page_content(content)
+      pages = split_into_pages(sanitized_content)
+      @page_count = pages.size
+
+      pages
+    end
+  end
+
+  def split_into_pages(content)
+    words = content.split(/\s+/)
+    pages = []
+    words.each_slice(WORDS_PER_PAGE) do |page_words|
+      pages << page_words.join(' ')
+    end
+    pages
+  end
+
+  # Sanitize the content
+  def sanitize_page_content(content)
+    sanitized_text = Sanitizer.remove_excessive_newlines(content)
+    sanitized_text = Sanitizer.remove_excessive_spaces(sanitized_text)
+    sanitized_text = Sanitizer.remove_bullet_points(sanitized_text)
+    # Add additional sanitization methods as required
+    sanitized_text
+  end
+end

data/lib/content_splitters/basic_sentence_splitter.rb

@@ -0,0 +1,45 @@
+require 'ruby/openai'
+
+class BasicTextChunker
+  def initialize(token_limit=390)
+    @token_limit = token_limit
+  end
+
+  def split_into_chunks(text)
+    sentences = text.split(/[.!?]\s+/)
+    chunks = []
+    current_chunk = ""
+    current_token_count = 0
+
+    sentences.each do |sentence|
+      sentence_token_count = OpenAI.rough_token_count(sentence)
+
+      while sentence_token_count > @token_limit
+        tokens_to_take = @token_limit - current_token_count
+        partial = sentence.split(/\s+/).first(tokens_to_take).join(" ")
+        current_chunk += partial + " "
+        sentence = sentence[partial.length..].strip
+        current_token_count += tokens_to_take
+        sentence_token_count -= tokens_to_take
+
+        if current_token_count == @token_limit
+          chunks << current_chunk.strip
+          current_chunk = ""
+          current_token_count = 0
+        end
+      end
+
+      if current_token_count + sentence_token_count <= @token_limit
+        current_chunk += sentence + " "
+        current_token_count += sentence_token_count
+      else
+        chunks << current_chunk.strip
+        current_chunk = sentence + " "
+        current_token_count = sentence_token_count
+      end
+    end
+
+    chunks << current_chunk.strip unless current_chunk.empty?
+    chunks
+  end
+end

data/lib/embedding_engines/mistral_embeddings.rb

@@ -0,0 +1,63 @@
+require 'dotenv'
+require_relative '../mistral_rb.rb'
+require_relative '../content_splitters/basic_sentence_splitter.rb'
+
+Dotenv.load()
+
+class MistralEmbeddingCreator
+  def initialize(api_key = nil, chunker = BasicTextChunker.new, model = "mistral-embed")
+    @chunker = chunker
+    @model = model
+    @api_key = api_key || ENV['MISTRAL_API_KEY']
+
+    if @api_key
+      @llm = MistralAPI.new(api_key: @api_key)
+    else
+      Rails.logger.error "MISTRAL AI API key not provided. Set the MISTRAL_API_KEY in the ENV variables or pass it as an argument."
+    end
+  end
+
+  def call(text, pages_mode=true)
+
+    if pages_mode
+      vectors = []
+      return [] unless @llm  # Return empty if the API client isn't set up
+
+      # Divide the text into chunks for each page
+      text.each_with_index do |page_content, page_index|
+        chunks = @chunker.split_into_chunks(page_content)
+
+        # Create embeddings for each chunk
+        chunks.each_with_index do |chunk, index|
+          response = @llm.create_embeddings(
+            model: @model,
+            input: [chunk]
+          )
+
+          # Extract the embeddings from the response
+          embedding = response.data.first.embedding
+
+          # Create vector data for the chunk and keep page numbers for reference
+          vector_data = {
+            id: "vec #{index + 1}",
+            values: embedding,
+            metadata: {
+                text: chunk,
+                page: page_index + 1,
+            }
+          }
+          # storing each chunk vector data in an array
+          vectors << vector_data
+        end
+      end
+      vectors
+    else
+      response = @llm.create_embeddings(
+        model: @model,
+        input: [text]
+      )
+
+      response.data.first.embedding
+    end
+  end
+end

data/lib/embedding_engines/openai_embeddings.rb

@@ -0,0 +1,67 @@
+require 'dotenv'
+require 'ruby/openai'
+require_relative '../content_splitters/basic_sentence_splitter.rb'
+
+Dotenv.load()
+
+class OpenaiEmbeddingCreator
+  def initialize(api_key = nil, chunker = BasicTextChunker.new, model = "text-embedding-ada-002")
+    @chunker = chunker
+    @model = model
+    @api_key = api_key || ENV['OPENAI_API_KEY']
+
+    if @api_key
+      @llm = OpenAI::Client.new(access_token: @api_key)
+    else
+      Rails.logger.error "OpenAI API key not provided. Set the OPENAI_API_KEY in the ENV variables or pass it as an argument."
+    end
+  end
+
+  def call(text, pages_mode=true)
+
+    if pages_mode
+      vectors = []
+      return [] unless @llm  # Return empty if the API client isn't set up
+
+      # Divide the text into chunks for each page
+      text.each_with_index do |page_content, page_index|
+        chunks = @chunker.split_into_chunks(page_content)
+
+        # Create embeddings for each chunk
+        chunks.each_with_index do |chunk, index|
+          response = @llm.embeddings(
+            parameters: {
+              model: @model,
+              input: chunk
+            }
+          )
+
+          # Extract the embeddings from the response
+          embedding = response['data'][0]['embedding']
+
+          # Create vector data for the chunk and keep page numbers for reference
+          vector_data = {
+            id: "vec #{index + 1}",
+            values: embedding,
+            metadata: {
+                text: chunk,
+                page: page_index + 1,
+            }
+          }
+          # storing each chunk vector data in an array
+          vectors << vector_data
+        end
+      end
+      vectors
+    else
+      response = @llm.embeddings(
+        parameters: {
+          model: @model,
+          input: chunk
+        }
+      )
+      # Extract the embeddings from the response
+      response['data'][0]['embedding']
+    end
+  end
+end

data/lib/mistral_rb/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module MistralRb
-  VERSION = "0.1.3"
+  VERSION = "0.1.4"
 end

data/lib/mistral_rb.rb

@@ -4,11 +4,14 @@ require_relative "mistral_rb/version"
 require "httparty"
 require "json"
 require_relative "mistral_rb/response_models"
+require 'dotenv'
+
+Dotenv.load()
 
 class MistralAPI
   include HTTParty
 
-  def initialize(api_key:, base_uri: "https://api.mistral.ai/v1")
+  def initialize(api_key: ENV["MISTRAL_API_KEY"], base_uri: "https://api.mistral.ai/v1")
     @headers = {
       "Authorization" => "Bearer #{api_key}",
       "Content-Type" => "application/json"

data/lib/responder.rb

@@ -0,0 +1,91 @@
+require 'dotenv'
+require 'ruby/openai'
+require_relative './mistral_rb.rb'
+require_relative './content_extractors/content_extractor_factory.rb'
+require_relative './content_splitters/basic_sentence_splitter.rb'
+require_relative './embedding_engines/mistral_embeddings.rb'
+require_relative './vector_stores/pinecone.rb'
+require_relative './utils/similarity_service.rb'
+require_relative './utils/adapters.rb'
+
+Dotenv.load()
+
+class Responder
+  def initialize(vector_store:, llm: MistralAPI.new, file:, embedding_creator: MistralEmbeddingCreator.new)
+    @vector_store = vector_store
+    @llm = llm
+    @file = file
+    @embedding_creator = embedding_creator
+  end
+
+  def call(question, top_k=10)
+    embedding = text_to_embedding(question)
+    results = process_similarity(question, top_k)
+    context = fetch_context(embedding, top_k)
+    merged_text = merge_texts(results, context)
+    prompt = construct_prompt(question, merged_text)
+    generate_response(prompt)
+  end
+
+  private
+
+  def extract_content
+    @extractor ||= ContentExtractorFactory.for(@file)
+
+    # Check if either @pages or @content is uninitialized
+    if @pages.nil? || @content.nil?
+      extracted_pages, extracted_content = @extractor.call
+      @pages ||= extracted_pages
+      @content ||= extracted_content
+    end
+  end
+
+  def store_embeddings
+    @embeddings ||= @embedding_creator.call(@pages)
+    @namespace ||= @vector_store.store(@embeddings, @content)
+  end
+
+  def text_to_embedding(question)
+    @embedding_creator.call(question, false)
+  end
+
+  # This method processes the similarity between the question and the content
+  def process_similarity(question, top_k)
+    extract_content # Ensure content is extracted
+    similarity_service = SimilarityService.new(question, @pages)
+    similarity_service.most_similar_sentences(top_k)
+  end
+
+  # Fetches context from the vector store based on the embedding
+  def fetch_context(embedding, top_k)
+    store_embeddings # Ensure embeddings are stored
+    if @namespace
+      @vector_store.index.query(
+        vector: embedding,
+        namespace: @namespace,
+        top_k: top_k,
+        include_values: false,
+        include_metadata: true
+      )
+    else
+      nil
+    end
+  end
+
+  # Merges the results from similarity processing with the context
+  def merge_texts(results, context)
+    [results, context].compact.join(' ')
+  end
+
+  def construct_prompt(question, merged_text)
+    "You are a helpful assistant. Answer this question: #{question}, using these information from the document the user uploaded: #{merged_text} in 60 words. Reply in the language of the question."
+  end
+
+  def generate_response(prompt)
+    response = @llm.create_chat_completion(
+      model: "mistral-tiny",
+      messages: [{role: "user", content: prompt}]
+    )
+    response.choices.first.message.content
+  end
+end

data/lib/utils/adapters.rb

@@ -0,0 +1,12 @@
+class LocalFileAdapter
+  # This adapter will wrap a local file path and provide a download method that yields the file's contents
+  def initialize(file_path)
+    @file_path = file_path
+  end
+
+  def download
+    File.open(@file_path, 'rb') do |file|
+      yield file.read
+    end
+  end
+end

data/lib/utils/sanitizer.rb

@@ -0,0 +1,16 @@
+module Sanitizer
+  # Remove sequences of more than two newline characters
+  def self.remove_excessive_newlines(text)
+    text.gsub(/(\n\s*){3,}/, "\n\n")
+  end
+
+  # Remove sequences of more than two spaces and replace with one space
+  def self.remove_excessive_spaces(text)
+    text.gsub(/ {3,}/, ' ')
+  end
+
+  # Remove bullet point characters
+  def self.remove_bullet_points(text)
+    text.gsub("•", "")
+  end
+end

data/lib/utils/similarity_service.rb

@@ -0,0 +1,40 @@
+
+class SimilarityService
+  FRENCH_STOP_WORDS = %w(
+    je tu il nous vous ils elle me te se le la les et ou mais
+    que quand donc or ni car
+  ).freeze
+
+  ENGLISH_STOP_WORDS = %w(
+    i you he we they she me him us them and or but that when so nor for
+  ).freeze
+
+  STOP_WORDS = (FRENCH_STOP_WORDS + ENGLISH_STOP_WORDS).freeze
+
+  def initialize(input_question, document_chunks)
+    @input_question = input_question
+    @document_chunks = document_chunks
+  end
+
+  def jaccard_similarity(str1, str2)
+    set1 = str1.downcase.split(" ").reject { |word| STOP_WORDS.include?(word) }.to_set
+    set2 = str2.downcase.split(" ").reject { |word| STOP_WORDS.include?(word) }.to_set
+    intersection = set1 & set2
+    union = set1 | set2
+    intersection.size.to_f / union.size
+  end
+
+  def most_similar_sentences(top_k)
+    sentence_delimiters = /[\.\?!:]/
+    all_sentences = @document_chunks.flat_map { |chunk| chunk.split(sentence_delimiters).map(&:strip) }
+
+    similarities = all_sentences.map do |sentence|
+      [sentence, jaccard_similarity(@input_question, sentence)]
+    end
+
+    # Sort by similarity and take the top_k
+    top_sentences = similarities.sort_by { |_, similarity| -similarity }.take(top_k).map(&:first)
+
+    top_sentences.join(' ')
+  end
+end

data/lib/vector_stores/pinecone.rb

@@ -0,0 +1,61 @@
+require 'pinecone'
+require 'digest'
+require 'dotenv'
+
+Dotenv.load()
+
+class PineconeService
+  attr_reader :index
+  def initialize(pinecone_key: ENV['PINECONE_API_KEY'], pinecone_env: ENV['PINECONE_ENV'], index_name:)
+    @pinecone_key = pinecone_key
+    @pinecone_env = pinecone_env
+    @index_name = index_name
+
+
+    Pinecone.configure do |config|
+      config.api_key  = @pinecone_key
+      config.environment = @pinecone_env
+    end
+
+    if @pinecone_key && @pinecone_env
+      @pinecone = Pinecone::Client.new
+    else
+      Rails.logger.error "Set the PINECONE_API_KEY and PINECONE_ENV in the ENV variables"
+    end
+    @index = @pinecone.index(@index_name)
+  end
+
+  def compute_hash(text)
+    Digest::SHA256.hexdigest(text)[0...44]
+  end
+
+  def store(embeddings, text)
+    namespace = compute_hash(text)
+
+    upsert_with_retry(@index, namespace, embeddings)
+
+    namespace
+  end
+
+  private
+
+  def upsert_with_retry(index, namespace, embeddings, max_retries = 5, retry_delay = 10)
+    retries = 0
+    response = nil
+
+    loop do
+      response = index.upsert(
+        namespace: namespace,
+        vectors: embeddings
+      )
+
+      break if response["code"] != 9 || retries >= max_retries
+
+      puts "Encountered error. Retrying in #{retry_delay} seconds... (Attempt #{retries + 1} of #{max_retries})"
+      sleep(retry_delay)
+      retries += 1
+    end
+
+    response
+  end
+end

data/mistral_rb.gemspec

@@ -31,10 +31,22 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   # Specify runtime and development dependencies in gemspec
-  spec.add_runtime_dependency "httparty", "~> 0.18"
   spec.add_development_dependency "bundler", "~> 2.0"
   spec.add_development_dependency "rake", "~> 13.0"
 
+  spec.add_runtime_dependency "httparty", "~> 0.18"
+  spec.add_runtime_dependency "mime-types"
+  spec.add_runtime_dependency "pdf-reader"
+  spec.add_runtime_dependency "pinecone"
+  spec.add_runtime_dependency "docx"
+  spec.add_runtime_dependency "dotenv-rails"
+  # spec.add_runtime_dependency "csv"
+  # spec.add_runtime_dependency "daru"
+  spec.add_runtime_dependency "nokogiri"
+  spec.add_runtime_dependency 'selenium-webdriver', '~> 4.5'
+  spec.add_runtime_dependency 'webdrivers', '~> 5.3'
+  spec.add_runtime_dependency 'watir'
+  spec.add_runtime_dependency 'ruby-openai'
   # For more information and examples about making a new gem, check out our
   # guide at: https://bundler.io/guides/creating_gem.html
 end

metadata

@@ -1,15 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: mistral_rb
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Franck Stephane Ndzomga
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-12-24 00:00:00.000000000 Z
+date: 2023-12-26 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: httparty
   requirement: !ruby/object:Gem::Requirement
@@ -25,33 +53,145 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.18'
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: mime-types
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pdf-reader
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pinecone
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: docx
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: dotenv-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: selenium-webdriver
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
-  type: :development
+        version: '4.5'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '4.5'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: webdrivers
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '13.0'
-  type: :development
+        version: '5.3'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '13.0'
+        version: '5.3'
+- !ruby/object:Gem::Dependency
+  name: watir
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-openai
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: This gem provides an easy-to-use interface for the Mistral AI API.
 email:
 - ndzomgafs@gmail.com
@@ -65,9 +205,22 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- lib/content_extractors/content_extractor_factory.rb
+- lib/content_extractors/docx_content_extractor.rb
+- lib/content_extractors/html_content_extractor.rb
+- lib/content_extractors/pdf_content_extractor.rb
+- lib/content_extractors/text_content_extractor.rb
+- lib/content_splitters/basic_sentence_splitter.rb
+- lib/embedding_engines/mistral_embeddings.rb
+- lib/embedding_engines/openai_embeddings.rb
 - lib/mistral_rb.rb
 - lib/mistral_rb/response_models.rb
 - lib/mistral_rb/version.rb
+- lib/responder.rb
+- lib/utils/adapters.rb
+- lib/utils/sanitizer.rb
+- lib/utils/similarity_service.rb
+- lib/vector_stores/pinecone.rb
 - mistral_rb.gemspec
 - sig/mistral_rb.rbs
 homepage: https://github.com/fsndzomga/mistral_rb
@@ -92,7 +245,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.5.3
 signing_key:
 specification_version: 4
 summary: A simple wrapper for the Mistral API