langchainrb 0.1.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 1124e5c3ad459df63509c1682a713ecdf3952904df4685a3e0583c4e3c3a79dc
+  data.tar.gz: 0ebb4c9c0774101b32f2f679f95b1c99d425462407d7b28bd07273f9610c42fe
+SHA512:
+  metadata.gz: f5c3fb67e7562df5b3e317f8f0c1d75e3948ce53812d10b5291236e4c068d8755781cc220add9eea686af189e77354b69daa45cb775353f3a3434dca1314920e
+  data.tar.gz: 0fb89fc2d4f1535b4af8ba5d3c9956da23ed3dd1174cb332f9c285838d9b1bfc3526552025ea1deca1b91e66275cef84b2c973c4b6118d3387074057ad192e67

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+## [Unreleased]
+
+## [0.1.0] - 2023-04-26
+
+- Initial release

data/Gemfile

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in langchain.gemspec
+gemspec
+
+gem "rake", "~> 13.0"
+
+gem "rspec", "~> 3.0"

data/Gemfile.lock

@@ -0,0 +1,179 @@
+PATH
+  remote: .
+  specs:
+    langchainrb (0.1.3)
+      cohere-ruby (~> 0.9.1)
+      milvus (~> 0.9.0)
+      pinecone (~> 0.1.6)
+      qdrant-ruby (~> 0.9.0)
+      ruby-openai (~> 4.0.0)
+      weaviate-ruby (~> 0.8.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    actionpack (7.0.4.3)
+      actionview (= 7.0.4.3)
+      activesupport (= 7.0.4.3)
+      rack (~> 2.0, >= 2.2.0)
+      rack-test (>= 0.6.3)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.2.0)
+    actionview (7.0.4.3)
+      activesupport (= 7.0.4.3)
+      builder (~> 3.1)
+      erubi (~> 1.4)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.1, >= 1.2.0)
+    activesupport (7.0.4.3)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+    builder (3.2.4)
+    byebug (11.1.3)
+    coderay (1.1.3)
+    cohere-ruby (0.9.1)
+      faraday (~> 2.7.0)
+    concurrent-ruby (1.2.2)
+    crass (1.0.6)
+    diff-lcs (1.5.0)
+    dotenv (2.7.6)
+    dotenv-rails (2.7.6)
+      dotenv (= 2.7.6)
+      railties (>= 3.2)
+    dry-configurable (1.0.1)
+      dry-core (~> 1.0, < 2)
+      zeitwerk (~> 2.6)
+    dry-core (1.0.0)
+      concurrent-ruby (~> 1.0)
+      zeitwerk (~> 2.6)
+    dry-inflector (1.0.0)
+    dry-initializer (3.1.1)
+    dry-logic (1.5.0)
+      concurrent-ruby (~> 1.0)
+      dry-core (~> 1.0, < 2)
+      zeitwerk (~> 2.6)
+    dry-schema (1.13.1)
+      concurrent-ruby (~> 1.0)
+      dry-configurable (~> 1.0, >= 1.0.1)
+      dry-core (~> 1.0, < 2)
+      dry-initializer (~> 3.0)
+      dry-logic (>= 1.4, < 2)
+      dry-types (>= 1.7, < 2)
+      zeitwerk (~> 2.6)
+    dry-struct (1.6.0)
+      dry-core (~> 1.0, < 2)
+      dry-types (>= 1.7, < 2)
+      ice_nine (~> 0.11)
+      zeitwerk (~> 2.6)
+    dry-types (1.7.1)
+      concurrent-ruby (~> 1.0)
+      dry-core (~> 1.0)
+      dry-inflector (~> 1.0)
+      dry-logic (~> 1.4)
+      zeitwerk (~> 2.6)
+    dry-validation (1.10.0)
+      concurrent-ruby (~> 1.0)
+      dry-core (~> 1.0, < 2)
+      dry-initializer (~> 3.0)
+      dry-schema (>= 1.12, < 2)
+      zeitwerk (~> 2.6)
+    erubi (1.12.0)
+    faraday (2.7.4)
+      faraday-net_http (>= 2.0, < 3.1)
+      ruby2_keywords (>= 0.0.4)
+    faraday-multipart (1.0.4)
+      multipart-post (~> 2)
+    faraday-net_http (3.0.2)
+    graphlient (0.7.0)
+      faraday (~> 2.0)
+      graphql-client
+    graphql (2.0.21)
+    graphql-client (0.18.0)
+      activesupport (>= 3.0)
+      graphql
+    httparty (0.21.0)
+      mini_mime (>= 1.0.0)
+      multi_xml (>= 0.5.2)
+    i18n (1.13.0)
+      concurrent-ruby (~> 1.0)
+    ice_nine (0.11.2)
+    loofah (2.20.0)
+      crass (~> 1.0.2)
+      nokogiri (>= 1.5.9)
+    method_source (1.0.0)
+    milvus (0.9.0)
+      faraday (~> 2.7.0)
+    mini_mime (1.1.2)
+    minitest (5.18.0)
+    multi_xml (0.6.0)
+    multipart-post (2.3.0)
+    nokogiri (1.14.3-x86_64-darwin)
+      racc (~> 1.4)
+    pinecone (0.1.71)
+      dry-struct (~> 1.6.0)
+      dry-validation (~> 1.10.0)
+      httparty (~> 0.21.0)
+    pry (0.14.2)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    pry-byebug (3.10.1)
+      byebug (~> 11.0)
+      pry (>= 0.13, < 0.15)
+    qdrant-ruby (0.9.0)
+      faraday (~> 2.7)
+    racc (1.6.2)
+    rack (2.2.7)
+    rack-test (2.1.0)
+      rack (>= 1.3)
+    rails-dom-testing (2.0.3)
+      activesupport (>= 4.2.0)
+      nokogiri (>= 1.6)
+    rails-html-sanitizer (1.5.0)
+      loofah (~> 2.19, >= 2.19.1)
+    railties (7.0.4.3)
+      actionpack (= 7.0.4.3)
+      activesupport (= 7.0.4.3)
+      method_source
+      rake (>= 12.2)
+      thor (~> 1.0)
+      zeitwerk (~> 2.5)
+    rake (13.0.6)
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.2)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.3)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.5)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.12.0)
+    rspec-support (3.12.0)
+    ruby-openai (4.0.0)
+      faraday (>= 1)
+      faraday-multipart (>= 1)
+    ruby2_keywords (0.0.5)
+    thor (1.2.1)
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
+    weaviate-ruby (0.8.0)
+      faraday (~> 2.7)
+      graphlient (~> 0.7.0)
+    zeitwerk (2.6.8)
+
+PLATFORMS
+  x86_64-darwin-19
+
+DEPENDENCIES
+  dotenv-rails (~> 2.7.6)
+  langchainrb!
+  pry-byebug (~> 3.10.0)
+  rake (~> 13.0)
+  rspec (~> 3.0)
+
+BUNDLED WITH
+   2.4.0

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2023 Andrei Bondarev
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,127 @@
+🦜️🔗 LangChain.rb
+--- 
+⚡ Building applications with LLMs through composability ⚡
+
+:warning: UNDER ACTIVE AND RAPID DEVELOPMENT (MAY BE BUGGY AND UNTESTED)
+
+![Tests status](https://github.com/andreibondarev/langchainrb/actions/workflows/ci.yml/badge.svg) [![Gem Version](https://badge.fury.io/rb/langchainrb.svg)](https://badge.fury.io/rb/langchainrb)
+
+Vectorsearch library is an abstraction layer on top of many popular vector search databases. It is a modern ORM that allows developers to easily chunk data, generate embeddings, store, search, query and retrieve data from vector search databases. Vectorsearch offers a straight-forward DSL and abstracts away overly complicated machine learning/data science-specific configurations and concepts
+
+## Installation
+
+Install the gem and add to the application's Gemfile by executing:
+
+    $ bundle add langchainrb
+
+If bundler is not being used to manage dependencies, install the gem by executing:
+
+    $ gem install langchainrb
+
+## Usage
+
+```ruby
+require "langchain"
+```
+
+List of currently supported vector search databases and features:
+
+| Database | Querying           | Storage | Schema Management | Backups | Rails Integration | ??? |
+| -------- |:------------------:| -------:| -----------------:| -------:| -----------------:| ---:|
+| Weaviate | :white_check_mark: | WIP     | WIP               | WIP     |                   |     |
+| Qdrant   | :white_check_mark: | WIP     | WIP               | WIP     |                   |     |
+| Milvus   | :white_check_mark: | WIP     | WIP               | WIP     |                   |     |
+| Pinecone | :white_check_mark: | WIP     | WIP               | WIP     |                   |     |
+
+### Using Vector Search Databases
+
+Choose the LLM provider you'll be using (OpenAI or Cohere) and retrieve the API key.
+
+Pick the vector search database you'll be using and instantiate the client:
+```ruby
+client = Vectorsearch::Weaviate.new(
+    url: ENV["WEAVIATE_URL"],
+    api_key: ENV["WEAVIATE_API_KEY"],
+    llm: :openai, # or :cohere
+    llm_api_key: ENV["OPENAI_API_KEY"]
+)
+
+# You can instantiate any other supported vector search database:
+client = Vectorsearch::Milvus.new(...)
+client = Vectorsearch::Qdrant.new(...)
+client = Vectorsearch::Pinecone.new(...)
+```
+
+```ruby
+# Creating the default schema
+client.create_default_schema
+```
+
+```ruby
+# Store your documents in your vector search database
+client.add_texts(
+    texts: [
+        "Begin by preheating your oven to 375°F (190°C). Prepare four boneless, skinless chicken breasts by cutting a pocket into the side of each breast, being careful not to cut all the way through. Season the chicken with salt and pepper to taste. In a large skillet, melt 2 tablespoons of unsalted butter over medium heat. Add 1 small diced onion and 2 minced garlic cloves, and cook until softened, about 3-4 minutes. Add 8 ounces of fresh spinach and cook until wilted, about 3 minutes. Remove the skillet from heat and let the mixture cool slightly.",
+        "In a bowl, combine the spinach mixture with 4 ounces of softened cream cheese, 1/4 cup of grated Parmesan cheese, 1/4 cup of shredded mozzarella cheese, and 1/4 teaspoon of red pepper flakes. Mix until well combined. Stuff each chicken breast pocket with an equal amount of the spinach mixture. Seal the pocket with a toothpick if necessary. In the same skillet, heat 1 tablespoon of olive oil over medium-high heat. Add the stuffed chicken breasts and sear on each side for 3-4 minutes, or until golden brown."
+    ]
+)
+```
+
+```ruby
+# Retrieve similar documents based on the query string passed in
+client.similarity_search(
+    query:,
+    k:       # number of results to be retrieved
+)
+```
+```ruby
+# Retrieve similar documents based on the embedding passed in
+client.similarity_search_by_vector(
+    embedding:,
+    k:       # number of results to be retrieved
+)
+```
+```ruby
+# Q&A-style querying based on the question passed in
+client.ask(
+    question:
+)
+```
+
+### Using Standalone LLMs
+
+#### OpenAI
+```ruby
+openai = LLM::OpenAI.new(api_key: ENV["OPENAI_API_KEY"])
+```
+```ruby
+openai.embed(text: "foo bar")
+```
+```ruby
+openai.complete(prompt: "What is the meaning of life?")
+```
+
+#### Cohere
+```ruby
+cohere = LLM::Cohere.new(api_key: ENV["COHERE_API_KEY"])
+```
+```ruby
+cohere.embed(text: "foo bar")
+```
+```ruby
+cohere.complete(prompt: "What is the meaning of life?")
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/andreibondarev/langchain.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/examples/store_and_query_with_pinecone.rb

@@ -0,0 +1,43 @@
+require "langchain"
+
+# Instantiate the Qdrant client
+pinecone = Vectorsearch::Pinecone.new(
+  environment: ENV["PINECONE_ENVIRONMENT"],
+  api_key: ENV["PINECONE_API_KEY"],
+  index_name: "recipes",
+  llm: :openai,
+  llm_api_key: ENV["OPENAI_API_KEY"]
+)
+
+# Create the default schema.
+pinecone.create_default_schema
+
+# Set up an array of text strings
+recipes = [
+  "Preheat oven to 400 degrees F (200 degrees C). Cut the top off the head of garlic. Arrange the garlic, carrots, celery, onion, pepper, and tomato on a large baking sheet in a single layer. Drizzle the olive oil over the vegetables; season with salt and pepper. Roast the vegetables in the preheated oven, turning every 20 minutes, until tender and browned, about 1 hour. Combine the water, thyme, parsley, and bay leaves in a large stock pot over medium-high heat. Squeeze the head of garlic into the stock pot, and discard the outer husk. Place the carrots, celery, onion, pepper, and tomato in the stock pot. Bring the water to a boil; reduce heat to low and simmer for 1 1/2 hours; strain and cool.",
+  "Heat oven to 190C/fan 170C/gas 5. Heat 1 tbsp oil and the butter in a frying pan, then add the onion and fry for 5 mins until softened. Cool slightly. Tip the sausagemeat, lemon zest, breadcrumbs, apricots, chestnuts and thyme into a bowl. Add the onion and cranberries, and mix everything together with your hands, adding plenty of pepper and a little salt. Cut each chicken breast into three fillets lengthwise and season all over with salt and pepper. Heat the remaining oil in the frying pan, and fry the chicken fillets quickly until browned, about 6-8 mins. Roll out two-thirds of the pastry to line a 20-23cm springform or deep loose-based tart tin. Press in half the sausage mix and spread to level. Then add the chicken pieces in one layer and cover with the rest of the sausage. Press down lightly. Roll out the remaining pastry. Brush the edges of the pastry with beaten egg and cover with the pastry lid. Pinch the edges to seal, then trim. Brush the top of the pie with egg, then roll out the trimmings to make holly leaf shapes and berries. Decorate the pie and brush again with egg. Set the tin on a baking sheet and bake for 50-60 mins, then cool in the tin for 15 mins. Remove and leave to cool completely. Serve with a winter salad and pickles."
+]
+
+# Add data to the index. Weaviate will use OpenAI to generate embeddings behind the scene.
+pinecone.add_texts(
+  texts: recipes
+)
+
+# Query your data
+pinecone.similarity_search(
+  query: "chicken",
+  k: 1
+)
+
+# Interact with your index through Q&A
+pinecone.ask(
+  question: "What is the best recipe for chicken?"
+)
+
+# Generate your an embedding and search by it
+openai = LLM::OpenAI.new(api_key: ENV['OPENAI_API_KEY'])
+embedding = openai.embed(text: "veggie")
+
+pinecone.similarity_search_by_vector(
+  embedding: embedding
+)

data/examples/store_and_query_with_qdrant.rb

@@ -0,0 +1,36 @@
+require "langchain"
+
+# Instantiate the Qdrant client
+qdrant = Vectorsearch::Qdrant.new(
+  url: ENV["QDRANT_URL"],
+  api_key: ENV["QDRANT_API_KEY"],
+  index_name: "recipes",
+  llm: :cohere,
+  llm_api_key: ENV["COHERE_API_KEY"]
+)
+
+
+# Create the default schema.
+qdrant.create_default_schema
+
+# Set up an array of text strings
+recipes = [
+  "Preheat oven to 400 degrees F (200 degrees C). Cut the top off the head of garlic. Arrange the garlic, carrots, celery, onion, pepper, and tomato on a large baking sheet in a single layer. Drizzle the olive oil over the vegetables; season with salt and pepper. Roast the vegetables in the preheated oven, turning every 20 minutes, until tender and browned, about 1 hour. Combine the water, thyme, parsley, and bay leaves in a large stock pot over medium-high heat. Squeeze the head of garlic into the stock pot, and discard the outer husk. Place the carrots, celery, onion, pepper, and tomato in the stock pot. Bring the water to a boil; reduce heat to low and simmer for 1 1/2 hours; strain and cool.",
+  "Heat oven to 190C/fan 170C/gas 5. Heat 1 tbsp oil and the butter in a frying pan, then add the onion and fry for 5 mins until softened. Cool slightly. Tip the sausagemeat, lemon zest, breadcrumbs, apricots, chestnuts and thyme into a bowl. Add the onion and cranberries, and mix everything together with your hands, adding plenty of pepper and a little salt. Cut each chicken breast into three fillets lengthwise and season all over with salt and pepper. Heat the remaining oil in the frying pan, and fry the chicken fillets quickly until browned, about 6-8 mins. Roll out two-thirds of the pastry to line a 20-23cm springform or deep loose-based tart tin. Press in half the sausage mix and spread to level. Then add the chicken pieces in one layer and cover with the rest of the sausage. Press down lightly. Roll out the remaining pastry. Brush the edges of the pastry with beaten egg and cover with the pastry lid. Pinch the edges to seal, then trim. Brush the top of the pie with egg, then roll out the trimmings to make holly leaf shapes and berries. Decorate the pie and brush again with egg. Set the tin on a baking sheet and bake for 50-60 mins, then cool in the tin for 15 mins. Remove and leave to cool completely. Serve with a winter salad and pickles."
+]
+
+# Add data to the index. Weaviate will use OpenAI to generate embeddings behind the scene.
+qdrant.add_texts(
+  texts: recipes
+)
+
+# Query your data
+qdrant.similarity_search(
+  query: "chicken",
+  k: 1
+)
+
+# Interact with your index through Q&A
+qdrant.ask(
+  question: "What is the best recipe for chicken?"
+)

data/examples/store_and_query_with_weaviate.rb

@@ -0,0 +1,30 @@
+require "langchain"
+
+# Instantiate the Weaviate client
+weaviate = Vectorsearch::Weaviate.new(
+  url: ENV["WEAVIATE_URL"],
+  api_key: ENV["WEAVIATE_API_KEY"],
+  index_name: "Recipes",
+  llm: :openai,
+  llm_api_key: ENV["OPENAI_API_KEY"]
+)
+
+# Create the default schema. A text field `content` will be used.
+weaviate.create_default_schema
+
+# Set up an array of text strings
+recipes = [
+  "Preheat oven to 400 degrees F (200 degrees C). Cut the top off the head of garlic. Arrange the garlic, carrots, celery, onion, pepper, and tomato on a large baking sheet in a single layer. Drizzle the olive oil over the vegetables; season with salt and pepper. Roast the vegetables in the preheated oven, turning every 20 minutes, until tender and browned, about 1 hour. Combine the water, thyme, parsley, and bay leaves in a large stock pot over medium-high heat. Squeeze the head of garlic into the stock pot, and discard the outer husk. Place the carrots, celery, onion, pepper, and tomato in the stock pot. Bring the water to a boil; reduce heat to low and simmer for 1 1/2 hours; strain and cool.",
+  "Heat oven to 190C/fan 170C/gas 5. Heat 1 tbsp oil and the butter in a frying pan, then add the onion and fry for 5 mins until softened. Cool slightly. Tip the sausagemeat, lemon zest, breadcrumbs, apricots, chestnuts and thyme into a bowl. Add the onion and cranberries, and mix everything together with your hands, adding plenty of pepper and a little salt. Cut each chicken breast into three fillets lengthwise and season all over with salt and pepper. Heat the remaining oil in the frying pan, and fry the chicken fillets quickly until browned, about 6-8 mins. Roll out two-thirds of the pastry to line a 20-23cm springform or deep loose-based tart tin. Press in half the sausage mix and spread to level. Then add the chicken pieces in one layer and cover with the rest of the sausage. Press down lightly. Roll out the remaining pastry. Brush the edges of the pastry with beaten egg and cover with the pastry lid. Pinch the edges to seal, then trim. Brush the top of the pie with egg, then roll out the trimmings to make holly leaf shapes and berries. Decorate the pie and brush again with egg. Set the tin on a baking sheet and bake for 50-60 mins, then cool in the tin for 15 mins. Remove and leave to cool completely. Serve with a winter salad and pickles."
+]
+
+# Add data to the index. Weaviate will use OpenAI to generate embeddings behind the scene.
+weaviate.add_texts(
+  texts: recipes
+)
+
+# Query your data
+weaviate.similarity_search(
+  query: "chicken",
+  k: 1
+)

data/lib/langchain.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require_relative "./version"
+
+module Vectorsearch
+  autoload :Base, "vectorsearch/base"
+  autoload :Milvus, "vectorsearch/milvus"
+  autoload :Pinecone, "vectorsearch/pinecone"
+  autoload :Qdrant, "vectorsearch/qdrant"
+  autoload :Weaviate, "vectorsearch/weaviate"
+end
+
+module LLM
+  autoload :Base, "llm/base"
+  autoload :Cohere, "llm/cohere"
+  autoload :OpenAI, "llm/openai"
+end

data/lib/llm/base.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module LLM
+  class Base
+    attr_reader :client
+
+    # Currently supported LLMs
+    # TODO: Add support for HuggingFace and other LLMs
+    LLMS = {
+      openai: "OpenAI",
+      cohere: "Cohere"
+    }.freeze
+
+    def default_dimension
+      self.class.const_get("DEFAULTS").dig(:dimension)
+    end
+  end
+end

data/lib/llm/cohere.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require "cohere"
+
+module LLM
+  class Cohere < Base
+
+    DEFAULTS = {
+      temperature: 0.0,
+      completion_model_name: "base",
+      embeddings_model_name: "small",
+      dimension: 1024
+    }.freeze
+
+    def initialize(api_key:)
+      @client = ::Cohere::Client.new(api_key: api_key)
+    end
+
+    # Generate an embedding for a given text
+    # @param text [String] The text to generate an embedding for
+    # @return [Hash] The embedding
+    def embed(text:)
+      response = client.embed(
+        texts: [text],
+        model: DEFAULTS[:embeddings_model_name],
+      )
+      response.dig("embeddings").first
+    end
+
+    # Generate a completion for a given prompt
+    # @param prompt [String] The prompt to generate a completion for
+    # @return [Hash] The completion
+    def complete(prompt:)
+      response = client.generate(
+        prompt: prompt,
+        temperature: DEFAULTS[:temperature],
+        model: DEFAULTS[:completion_model_name],
+      )
+      response.dig("generations").first.dig("text")
+    end
+
+    alias_method :generate_completion, :complete
+    alias_method :generate_embedding, :embed
+  end
+end

data/lib/llm/openai.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require "openai"
+
+module LLM
+  class OpenAI < Base
+
+    DEFAULTS = {
+      temperature: 0.0,
+      completion_model_name: "text-davinci-003",
+      embeddings_model_name: "text-embedding-ada-002",
+      dimension: 1536
+    }.freeze
+
+    def initialize(api_key:)
+      # TODO: Add support to pass `organization_id:`
+      @client = ::OpenAI::Client.new(access_token: api_key)
+    end
+
+    # Generate an embedding for a given text
+    # @param text [String] The text to generate an embedding for
+    # @return [Array] The embedding
+    def embed(text:)
+      response = client.embeddings(
+        parameters: {
+          model: DEFAULTS[:embeddings_model_name],
+          input: text
+        }
+      )
+      response.dig("data").first.dig("embedding")
+    end
+
+    # Generate a completion for a given prompt
+    # @param prompt [String] The prompt to generate a completion for
+    # @return [String] The completion
+    def complete(prompt:)
+      response = client.completions(
+        parameters: {
+          model: DEFAULTS[:completion_model_name],
+          temperature: DEFAULTS[:temperature],
+          prompt: prompt
+        }
+      )
+      response.dig("choices").first.dig("text")
+    end
+
+    alias_method :generate_completion, :complete
+    alias_method :generate_embedding, :embed
+  end
+end

data/lib/vectorsearch/base.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Vectorsearch
+  class Base
+    extend Forwardable
+
+    attr_reader :client, :index_name, :llm, :llm_api_key, :llm_client
+
+    DEFAULT_METRIC = "cosine".freeze
+
+    # @param llm [Symbol] The LLM to use
+    # @param llm_api_key [String] The API key for the LLM
+    def initialize(llm:, llm_api_key:)
+      validate_llm!(llm: llm)
+
+      @llm = llm
+      @llm_api_key = llm_api_key
+
+      @llm_client = LLM.const_get(LLM::Base::LLMS.fetch(llm)).new(api_key: llm_api_key)
+    end
+
+    def create_default_schema
+      raise NotImplementedError
+    end
+
+    def add_texts(texts:)
+      raise NotImplementedError
+    end
+
+    # NotImplementedError will be raised if the subclass does not implement this method
+    def ask(question:)
+      raise NotImplementedError
+    end
+
+    def_delegators :llm_client,
+      :generate_embedding,
+      :generate_completion,
+      :default_dimension
+
+    # def generate_embedding(text:)
+    #   llm_client.embed(text: text)
+    # end
+
+    # def generate_completion(prompt:)
+    #   llm_client.complete(prompt: prompt)
+    # end
+
+    # def default_dimension
+    #   llm_client.default_dimension
+    # end
+
+    def generate_prompt(question:, context:)
+      "Context:\n" +
+      "#{context}\n" +
+      "---\n" +
+      "Question: #{question}\n" +
+      "---\n" +
+      "Answer:"
+    end
+
+    private
+
+    def validate_llm!(llm:)
+      # TODO: Fix that this works is string `llm` value is passed in instead of symbol
+      unless LLM::Base::LLMS.keys.include?(llm)
+        raise ArgumentError, "LLM must be one of #{LLMS}"
+      end
+    end
+  end
+end

data/lib/vectorsearch/milvus.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require "milvus"
+
+module Vectorsearch
+  class Milvus < Base
+    def initialize(
+      url:,
+      api_key: nil,
+      index_name:,
+      llm:,
+      llm_api_key:
+    )
+      @client = ::Milvus::Client.new(
+        url: url
+      )
+      @index_name = index_name
+
+      super(llm: llm, llm_api_key: llm_api_key)
+    end
+
+    def add_texts(
+      texts:
+    )
+      client.entities.insert(
+        collection_name: index_name,
+        num_rows: texts.count,
+        fields_data: [
+          {
+            field_name: "content",
+            type: ::Milvus::DATA_TYPES["varchar"],
+            field: texts
+          }, {
+            field_name: "vectors",
+            type: ::Milvus::DATA_TYPES["binary_vector"],
+            field: texts.map { |text| generate_embedding(text: text) }
+          }
+        ]
+      )
+    end
+
+    # Create default schema
+    # @return [Hash] The response from the server
+    def create_default_schema
+      client.collections.create(
+        auto_id: true,
+        collection_name: index_name,
+        description: "Default schema created by Vectorsearch",
+        fields: [
+          {
+            name: "id",
+            is_primary_key: true,
+            autoID: true,
+            data_type: ::Milvus::DATA_TYPES["int64"]
+          }, {
+            name: "content",
+            is_primary_key: false,
+            data_type: ::Milvus::DATA_TYPES["varchar"],
+            type_params: [
+              {
+                key: "max_length",
+                value: "32768" # Largest allowed value
+              }
+            ]
+          }, {
+            name: "vectors",
+            data_type: ::Milvus::DATA_TYPES["binary_vector"],
+            is_primary_key: false,
+            type_params: [
+              {
+                key: "dim",
+                value: default_dimension.to_s
+              }
+            ]
+          }
+        ]
+      )
+    end
+
+    def similarity_search(
+      query:,
+      k: 4
+    )
+      embedding = generate_embedding(text: query)
+
+      similarity_search_by_vector(
+        embedding: embedding,
+        k: k
+      )
+    end
+
+    def similarity_search_by_vector(
+      embedding:,
+      k: 4
+    )
+      client.search(
+        collection_name: index_name,
+        top_k: k.to_s,
+        vectors: [ embedding ],
+        dsl_type: 1,
+        params: "{\"nprobe\": 10}",
+        anns_field: "content",
+        metric_type: "L2"
+      )
+    end
+
+    def ask(question:)
+      raise NotImplementedError
+    end
+  end
+end

data/lib/vectorsearch/pinecone.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+require "pinecone"
+
+module Vectorsearch
+  class Pinecone < Base
+    # Initialize the Pinecone client
+    # @param environment [String] The environment to use
+    # @param api_key [String] The API key to use
+    # @param index_name [String] The name of the index to use
+    # @param llm [Symbol] The LLM to use
+    # @param llm_api_key [String] The API key for the LLM
+    def initialize(
+      environment:,
+      api_key:,
+      index_name:,
+      llm:,
+      llm_api_key:
+    )
+      ::Pinecone.configure do |config|
+        config.api_key  = api_key
+        config.environment = environment
+      end
+    
+      @client = ::Pinecone::Client.new
+      @index_name = index_name
+
+      super(llm: llm, llm_api_key: llm_api_key)
+    end
+
+    # Add a list of texts to the index
+    # @param texts [Array] The list of texts to add
+    # @return [Hash] The response from the server
+    def add_texts(
+      texts:
+    )
+      vectors = texts.map do |text|
+        {
+          # TODO: Allows passing in your own IDs
+          id: SecureRandom.uuid,
+          metadata: { content: text },
+          values: generate_embedding(text: text)
+        }
+      end
+
+      index = client.index(index_name)
+
+      index.upsert(vectors: vectors)
+    end
+
+    # Create the index with the default schema
+    # @return [Hash] The response from the server
+    def create_default_schema
+      client.create_index(
+        metric: DEFAULT_METRIC,
+        name: index_name,
+        dimension: default_dimension
+      )
+    end
+
+    # Search for similar texts
+    # @param query [String] The text to search for
+    # @param k [Integer] The number of results to return
+    # @return [Array] The list of results
+    def similarity_search(
+      query:,
+      k: 4
+    )
+      embedding = generate_embedding(text: query)
+
+      similarity_search_by_vector(
+        embedding: embedding,
+        k: k
+      )
+    end
+
+    # Search for similar texts by embedding
+    # @param embedding [Array] The embedding to search for
+    # @param k [Integer] The number of results to return
+    # @return [Array] The list of results
+    def similarity_search_by_vector(
+      embedding:,
+      k: 4
+    )
+      index = client.index(index_name)
+
+      response = index.query(
+        vector: embedding,
+        top_k: k,
+        include_values: true,
+        include_metadata: true
+      )
+      response.dig("matches")
+    end
+
+    # Ask a question and return the answer
+    # @param question [String] The question to ask
+    # @return [String] The answer to the question
+    def ask(question:)
+      search_results = similarity_search(query: question)
+
+      context = search_results.map do |result|
+        result.dig("metadata").to_s
+      end
+      context = context.join("\n---\n")
+
+      prompt = generate_prompt(question: question, context: context)
+
+      generate_completion(prompt: prompt)
+    end
+  end
+end

data/lib/vectorsearch/qdrant.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require "qdrant"
+
+module Vectorsearch
+  class Qdrant < Base
+    # Initialize the Qdrant client
+    # @param url [String] The URL of the Qdrant server
+    # @param api_key [String] The API key to use
+    # @param index_name [String] The name of the index to use
+    # @param llm [Symbol] The LLM to use
+    # @param llm_api_key [String] The API key for the LLM
+    def initialize(
+      url:,
+      api_key:,
+      index_name:,
+      llm:,
+      llm_api_key:
+    )
+      @client = ::Qdrant::Client.new(
+        url: url,
+        api_key: api_key
+      )
+      @index_name = index_name
+
+      super(llm: llm, llm_api_key: llm_api_key)
+    end
+
+    # Add a list of texts to the index
+    # @param texts [Array] The list of texts to add
+    # @return [Hash] The response from the server
+    def add_texts(
+      texts:
+    )
+      batch = { ids: [], vectors: [], payloads: [] }
+
+      texts.each do |text|
+        batch[:ids].push(SecureRandom.uuid)
+        batch[:vectors].push(generate_embedding(text: text))
+        batch[:payloads].push({ content: text })
+      end
+
+      client.points.upsert(
+        collection_name: index_name,
+        batch: batch
+      )
+    end
+
+    # Create the index with the default schema
+    # @return [Hash] The response from the server
+    def create_default_schema
+      client.collections.create(
+        collection_name: index_name,
+        vectors: {
+          distance: DEFAULT_METRIC.capitalize,
+          size: default_dimension
+        }
+      )
+    end
+
+    # Search for similar texts
+    # @param query [String] The text to search for
+    # @param k [Integer] The number of results to return
+    # @return [Hash] The response from the server
+    def similarity_search(
+      query:,
+      k: 4
+    )
+      embedding = generate_embedding(text: query)
+
+      similarity_search_by_vector(
+        embedding: embedding,
+        k: k
+      )
+    end
+
+    # Search for similar texts by embedding
+    # @param embedding [Array] The embedding to search for
+    # @param k [Integer] The number of results to return
+    # @return [Hash] The response from the server
+    def similarity_search_by_vector(
+      embedding:,
+      k: 4
+    )
+      client.points.search(
+        collection_name: index_name,
+        limit: k,
+        vector: embedding,
+        with_payload: true
+      )
+    end
+
+    # Ask a question and return the answer
+    # @param question [String] The question to ask
+    # @return [String] The answer to the question
+    def ask(question:)
+      search_results = similarity_search(query: question)
+
+      context = search_results.dig("result").map do |result|
+        result.dig("payload").to_s
+      end
+      context = context.join("\n---\n")
+
+      prompt = generate_prompt(question: question, context: context)
+
+      generate_completion(prompt: prompt)
+    end
+  end
+end

data/lib/vectorsearch/weaviate.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+require "weaviate"
+
+module Vectorsearch
+  class Weaviate < Base
+    # Initialize the Weaviate adapter
+    # @param url [String] The URL of the Weaviate instance
+    # @param api_key [String] The API key to use
+    # @param index_name [String] The name of the index to use
+    # @param llm [Symbol] The LLM to use
+    # @param llm_api_key [String] The API key for the LLM
+    def initialize(
+      url:,
+      api_key:,
+      index_name:,
+      llm:,
+      llm_api_key:
+    )
+      @client = ::Weaviate::Client.new(
+        url: url,
+        api_key: api_key,
+        model_service: llm,
+        model_service_api_key: llm_api_key
+      )
+      @index_name = index_name
+
+      super(llm: llm, llm_api_key: llm_api_key)
+    end
+
+    # Add a list of texts to the index
+    # @param texts [Array] The list of texts to add
+    # @return [Hash] The response from the server
+    def add_texts(
+      texts:
+    )
+      objects = texts.map do |text|
+        {
+          class: index_name,
+          properties: { content: text }
+        }
+      end
+
+      client.objects.batch_create(
+        objects: objects
+      )
+    end
+
+    # Create default schema
+    def create_default_schema
+      client.schema.create(
+        class_name: index_name,
+        vectorizer: "text2vec-#{llm.to_s}",
+        # TODO: Figure out a way to optionally enable it
+        # "module_config": {
+        #   "qna-openai": {}
+        # },
+        properties: [
+          # TODO: Allow passing in your own IDs
+          {
+            dataType: ["text"],
+            name: "content"
+          }
+        ]
+      )
+    end
+
+    # Return documents similar to the query
+    # @param query [String] The query to search for
+    # @param k [Integer|String] The number of results to return
+    # @return [Hash] The search results
+    def similarity_search(
+      query:,
+      k: 4
+    )
+      near_text = "{ concepts: [\"#{query}\"] }"
+
+      client.query.get(
+        class_name: index_name,
+        near_text: near_text,
+        limit: k.to_s,
+        fields: "content _additional { id }"
+      )
+    end
+
+    # Return documents similar to the vector
+    # @param embedding [Array] The vector to search for
+    # @param k [Integer|String] The number of results to return
+    # @return [Hash] The search results
+    def similarity_search_by_vector(
+      embedding:,
+      k: 4
+    )
+      near_vector = "{ vector: #{embedding} }"
+
+      client.query.get(
+        class_name: index_name,
+        near_vector: near_vector,
+        limit: k.to_s,
+        fields: "content _additional { id }"
+      )
+    end
+
+    # Ask a question and return the answer
+    # @param question [String] The question to ask
+    # @return [Hash] The answer
+    def ask(
+      question:
+    )
+      # Weaviate currently supports the `ask:` parameter only for the OpenAI LLM (with `qna-openai` module enabled).
+      # The Cohere support is on the way: https://github.com/weaviate/weaviate/pull/2600
+      if llm == :openai
+        ask_object = "{ question: \"#{question}\" }"
+
+        client.query.get(
+          class_name: index_name,
+          ask: ask_object,
+          limit: "1",
+          fields: "_additional { answer { result } }"
+        )
+      elsif llm == :cohere
+        search_results = similarity_search(query: question)
+
+        context = search_results.map do |result|
+          result.dig("content").to_s
+        end
+        context = context.join("\n---\n")
+
+        prompt = generate_prompt(question: question, context: context)
+
+        generate_completion(prompt: prompt)
+      end
+    end
+  end
+end

data/lib/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Langchain
+  VERSION = "0.1.3"
+end

data/sig/langchain.rbs

@@ -0,0 +1,4 @@
+module Langchain
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end

metadata

@@ -0,0 +1,180 @@
+--- !ruby/object:Gem::Specification
+name: langchainrb
+version: !ruby/object:Gem::Version
+  version: 0.1.3
+platform: ruby
+authors:
+- Andrei Bondarev
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2023-05-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.10.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.10.0
+- !ruby/object:Gem::Dependency
+  name: dotenv-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.7.6
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.7.6
+- !ruby/object:Gem::Dependency
+  name: cohere-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.1
+- !ruby/object:Gem::Dependency
+  name: milvus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.0
+- !ruby/object:Gem::Dependency
+  name: pinecone
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.6
+- !ruby/object:Gem::Dependency
+  name: ruby-openai
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.0.0
+- !ruby/object:Gem::Dependency
+  name: qdrant-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.0
+- !ruby/object:Gem::Dependency
+  name: weaviate-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+description: Build ML/AI-powered applications with Ruby's LangChain
+email:
+- andrei.bondarev13@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rspec"
+- CHANGELOG.md
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- examples/.keep
+- examples/store_and_query_with_pinecone.rb
+- examples/store_and_query_with_qdrant.rb
+- examples/store_and_query_with_weaviate.rb
+- lib/langchain.rb
+- lib/llm/base.rb
+- lib/llm/cohere.rb
+- lib/llm/openai.rb
+- lib/vectorsearch/base.rb
+- lib/vectorsearch/milvus.rb
+- lib/vectorsearch/pinecone.rb
+- lib/vectorsearch/qdrant.rb
+- lib/vectorsearch/weaviate.rb
+- lib/version.rb
+- sig/langchain.rbs
+homepage: https://rubygems.org/gems/langchainrb
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://rubygems.org/gems/langchainrb
+  source_code_uri: https://github.com/andreibondarev/langchainrb
+  changelog_uri: https://github.com/andreibondarev/langchainrb/CHANGELOG.md
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.6.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.3
+signing_key:
+specification_version: 4
+summary: Build ML/AI-powered applications with Ruby's LangChain
+test_files: []