boxcars 0.2.10 → 0.2.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b7d76e1223cfa17d11ab3670ebc6d43a8dbdc142e6aa85bd6bb6c230ccdea160
-  data.tar.gz: 39172ff71234851bcef10c762ba29481b4dba199e851c37e8092794ed24604b1
+  metadata.gz: da3d8b9838602151837c0cc5bb9f3cae841ba24d1c338eade82c7807d913d4bb
+  data.tar.gz: f7be434c18f0ff2c95625fe32fae25f3a5df265331425d0c1f0430ab75761578
 SHA512:
-  metadata.gz: b720092a75593e767564234f5990ccb7e57a382f7e5065c5cbaf9496ca0622ceb9dc63f07c10ff1ba305672c42817f94b371fadc3562b0043cfcde83ef2f1a8d
-  data.tar.gz: 2ded919f3e0157d777b541589d48472b0cceb348f1f9bfeb18f63b29dad25a4d23183a34aa111db726c366bbe0b74283d34a1c0e14372b726448e9c69961dc65
+  metadata.gz: 57dd238c56f13f63a4665d4469efdabfa5f3c54f82cb6832c4158858d4b307a80c57f619633cdad6934d64186d560dcab7a62efa9adc727edfa61afbc5acc188
+  data.tar.gz: d2c782acf20c6b6b13cbfadf8f5406363b347be90a058626ec1bb21fe32baf1acb57a4a72c4770a7ad820700b465c0474a498080604e23e5d0270001d5d4aec1

data/CHANGELOG.md

@@ -1,13 +1,25 @@
 # Changelog
 
-## [Unreleased](https://github.com/BoxcarsAI/boxcars/tree/HEAD)
+## [v0.2.10](https://github.com/BoxcarsAI/boxcars/tree/v0.2.10) (2023-05-05)
 
-[Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.2.9...HEAD)
+[Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.2.9...v0.2.10)
 
 **Implemented enhancements:**
 
 - Notion Q&A [\#13](https://github.com/BoxcarsAI/boxcars/issues/13)
 
+**Closed issues:**
+
+- undefined method `default\_train' for Boxcars:Module \(ActiveRecord example\) [\#66](https://github.com/BoxcarsAI/boxcars/issues/66)
+- Chore: reduce the number of markdown files in Notion DB folder [\#56](https://github.com/BoxcarsAI/boxcars/issues/56)
+
+**Merged pull requests:**
+
+- \[DRAFT\] Feature - add in memory vector store [\#65](https://github.com/BoxcarsAI/boxcars/pull/65) ([jaigouk](https://github.com/jaigouk))
+- Chore - rename module name from Embeddings to VectorStores [\#63](https://github.com/BoxcarsAI/boxcars/pull/63) ([jaigouk](https://github.com/jaigouk))
+- remove bunch of markdown files in Notion\_DB directory [\#62](https://github.com/BoxcarsAI/boxcars/pull/62) ([jaigouk](https://github.com/jaigouk))
+- Fixed typo in README.md [\#61](https://github.com/BoxcarsAI/boxcars/pull/61) ([robmack](https://github.com/robmack))
+
 ## [v0.2.9](https://github.com/BoxcarsAI/boxcars/tree/v0.2.9) (2023-04-22)
 
 [Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.2.8...v0.2.9)

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    boxcars (0.2.10)
+    boxcars (0.2.11)
       google_search_results (~> 2.2)
       gpt4all (~> 0.0.4)
       ruby-openai (~> 3.0)

data/lib/boxcars/boxcar.rb

@@ -156,4 +156,4 @@ require "boxcars/boxcar/wikipedia_search"
 require "boxcars/boxcar/sql"
 require "boxcars/boxcar/swagger"
 require "boxcars/boxcar/active_record"
-require "boxcars/boxcar/vector_search"
+require "boxcars/vector_search"

data/lib/boxcars/{boxcar/vector_search.rb → vector_search.rb}

@@ -8,4 +8,4 @@ module Boxcars
   end
 end
 
-require "boxcars/boxcar/vector_store"
+require "boxcars/vector_store"

data/lib/boxcars/{boxcar/vector_stores → vector_store}/document.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Boxcars
-  module VectorStores
+  module VectorStore
     class Document
       attr_accessor :page_content, :metadata
 

data/lib/boxcars/{boxcar/vector_stores → vector_store}/embed_via_open_ai.rb

@@ -3,7 +3,7 @@
 require 'openai'
 
 module Boxcars
-  module VectorStores
+  module VectorStore
     class EmbedViaOpenAI
       include VectorStore
 

data/lib/boxcars/{boxcar/vector_stores → vector_store}/embed_via_tensorflow.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Boxcars
-  module VectorStores
+  module VectorStore
     class EmbedViaTensorflow
       include VectorStore
       def call

data/lib/boxcars/{boxcar/vector_stores → vector_store}/hnswlib/build_vector_store.rb

@@ -5,7 +5,7 @@ require 'hnswlib'
 require 'json'
 
 module Boxcars
-  module VectorStores
+  module VectorStore
     module Hnswlib
       class BuildVectorStore
         include VectorStore
@@ -76,7 +76,7 @@ module Boxcars
 
           docs = []
           data.each do |chunk|
-            doc_output = Boxcars::VectorStores::SplitText.call(
+            doc_output = Boxcars::VectorStore::SplitText.call(
               separator: "\n", chunk_size: split_chunk_size, chunk_overlap: 0, text: chunk
             )
             docs.concat(doc_output)
@@ -98,7 +98,7 @@ module Boxcars
 
           puts "Initializing Store..."
           openai_client = Openai.open_ai_client
-          embeddings_with_dim = Boxcars::VectorStores::EmbedViaOpenAI.call(texts: documents, client: openai_client)
+          embeddings_with_dim = Boxcars::VectorStore::EmbedViaOpenAI.call(texts: documents, client: openai_client)
           document_embeddings = embeddings_with_dim.map.with_index do |item, index|
             { doc_id: index, embedding: item[:embedding], document: documents[index] }
           end
@@ -110,7 +110,7 @@ module Boxcars
           return true unless rebuild_required?
 
           puts "Saving Vectorstore"
-          Boxcars::VectorStores::Hnswlib::SaveToHnswlib.call(
+          Boxcars::VectorStore::Hnswlib::SaveToHnswlib.call(
             document_embeddings: embeddings_with_config[:document_embeddings],
             index_file_path: index_file_path,
             json_doc_file_path: json_doc_file_path,
@@ -121,7 +121,7 @@ module Boxcars
 
         def hnswlib_config(dim)
           # dim: length of datum point vector that will be indexed.
-          Boxcars::VectorStores::Hnswlib::HnswlibConfig.new(
+          Boxcars::VectorStore::Hnswlib::HnswlibConfig.new(
             metric: "l2", max_item: 10000, dim: dim
           )
         end

data/lib/boxcars/{boxcar/vector_stores → vector_store}/hnswlib/hnswlib_config.rb

@@ -3,7 +3,7 @@
 require 'json'
 
 module Boxcars
-  module VectorStores
+  module VectorStore
     module Hnswlib
       class HnswlibConfig
         attr_reader :metric, :max_item, :dim, :ef_construction, :m

data/lib/boxcars/{boxcar/vector_stores → vector_store}/hnswlib/hnswlib_search.rb

@@ -4,7 +4,7 @@ require 'hnswlib'
 require 'json'
 
 module Boxcars
-  module VectorStores
+  module VectorStore
     module Hnswlib
       class HnswlibSearch
         def initialize(vector_store:, options: {})

data/lib/boxcars/{boxcar/vector_stores → vector_store}/hnswlib/save_to_hnswlib.rb

@@ -5,14 +5,14 @@ require 'json'
 require 'fileutils'
 
 module Boxcars
-  module VectorStores
+  module VectorStore
     module Hnswlib
       class SaveToHnswlib
         include VectorStore
 
         # @param document_embeddings [Array] An array of hashes containing the document id, document text, and embedding.
         # @param index_file_path [String] The path to the index file.
-        # @param hnswlib_config [Boxcars::VectorStores::Hnswlib::Config] The config object for the hnswlib index.
+        # @param hnswlib_config [Boxcars::VectorStore::Hnswlib::Config] The config object for the hnswlib index.
         # @option json_doc_file_path [String] Optional. The path to the json file containing the document text.
         def initialize(document_embeddings:, index_file_path:, hnswlib_config:, json_doc_file_path: nil)
           @document_embeddings = document_embeddings

data/lib/boxcars/{boxcar/vector_stores → vector_store}/in_memory/add_documents.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Boxcars
-  module VectorStores
+  module VectorStore
     module InMemory
       MemoryVector = Struct.new(:content, :embedding, :metadatax)
 
@@ -49,9 +49,9 @@ module Boxcars
           @embeddings_method ||=
             case @embedding_tool
             when :openai
-              { klass: Boxcars::VectorStores::EmbedViaOpenAI, client: openai_client }
+              { klass: Boxcars::VectorStore::EmbedViaOpenAI, client: openai_client }
             when :tensorflow
-              { klass: Boxcars::VectorStores::EmbedViaTensorflow, client: nil }
+              { klass: Boxcars::VectorStore::EmbedViaTensorflow, client: nil }
             end
         end
 

data/lib/boxcars/{boxcar/vector_stores → vector_store}/in_memory/search.rb

@@ -9,14 +9,14 @@
 #     { page_content: "what's this", metadata: { a: 1 } },
 # ]
 #
-# vector_documents = Boxcars::VectorStores::InMemory::AddDocuments.call(embedding_tool: :openai, documents: documents)
+# vector_documents = Boxcars::VectorStore::InMemory::AddDocuments.call(embedding_tool: :openai, documents: documents)
 #
-# result = Boxcars::VectorStores::InMemory::Search.call(vecotr_documents: vector_documents, query: "hello")
+# result = Boxcars::VectorStore::InMemory::Search.call(vecotr_documents: vector_documents, query: "hello")
 #
-# expect(result).to eq(Boxcars::VectorStores::Document.new({ page_content: "hello", metadata: { a: 1 } }))
+# expect(result).to eq(Boxcars::VectorStore::Document.new({ page_content: "hello", metadata: { a: 1 } }))
 
 module Boxcars
-  module VectorStores
+  module VectorStore
     module InMemory
       class Search
         include VectorStore
@@ -63,9 +63,9 @@ module Boxcars
         def embeddings_method(embedding_tool)
           case embedding_tool
           when :openai
-            { klass: Boxcars::VectorStores::EmbedViaOpenAI, client: openai_client }
+            { klass: Boxcars::VectorStore::EmbedViaOpenAI, client: openai_client }
           when :tensorflow
-            { klass: Boxcars::VectorStores::EmbedViaTensorflow, client: nil }
+            { klass: Boxcars::VectorStore::EmbedViaTensorflow, client: nil }
           end
         end
 

data/lib/boxcars/{boxcar/vector_stores → vector_store}/similarity_search.rb

@@ -3,7 +3,7 @@
 require 'hnswlib'
 
 module Boxcars
-  module VectorStores
+  module VectorStore
     class SimilaritySearch
       def initialize(embeddings:, vector_store:, openai_connection: nil, openai_access_token: nil)
         @embeddings = embeddings
@@ -32,13 +32,13 @@ module Boxcars
       end
 
       def convert_query_to_vector(query)
-        Boxcars::VectorStores::EmbedViaOpenAI.call(texts: [query], client: openai_connection).first[:embedding]
+        Boxcars::VectorStore::EmbedViaOpenAI.call(texts: [query], client: openai_connection).first[:embedding]
       end
 
       def create_similarity_search_instance
         case vector_store
         when ::Hnswlib::HierarchicalNSW
-          Boxcars::VectorStores::Hnswlib::HnswlibSearch.new(
+          Boxcars::VectorStore::Hnswlib::HnswlibSearch.new(
             vector_store: vector_store,
             options: { json_doc_path: embeddings, num_neighbors: 2 }
           )

data/lib/boxcars/{boxcar/vector_stores → vector_store}/split_text.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Boxcars
-  module VectorStores
+  module VectorStore
     # Split a text into chunks of a given size.
     class SplitText
       include VectorStore

data/lib/boxcars/vector_store.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+# Boxcars is a framework for running a series of tools to get an answer to a question.
+module Boxcars
+  module VectorStore
+    module ClassMethods
+      VectorStoreError = Class.new(StandardError)
+
+      def call(*args, **kw_args)
+        new(*args, **kw_args).call
+      end
+    end
+
+    def self.included(base)
+      base.extend(ClassMethods)
+
+      class << base
+        private :new
+      end
+    end
+  end
+end
+
+require_relative "vector_store/document"
+require_relative "vector_store/embed_via_open_ai"
+require_relative "vector_store/embed_via_tensorflow"
+require_relative "vector_store/split_text"
+require_relative "vector_store/similarity_search"
+require_relative "vector_store/hnswlib/hnswlib_config"
+require_relative "vector_store/hnswlib/save_to_hnswlib"
+require_relative "vector_store/hnswlib/build_vector_store"
+require_relative "vector_store/hnswlib/hnswlib_search"
+require_relative "vector_store/in_memory/add_documents"
+require_relative "vector_store/in_memory/search"

data/lib/boxcars/version.rb

@@ -2,5 +2,5 @@
 
 module Boxcars
   # The current version of the gem.
-  VERSION = "0.2.10"
+  VERSION = "0.2.11"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: boxcars
 version: !ruby/object:Gem::Version
-  version: 0.2.10
+  version: 0.2.11
 platform: ruby
 authors:
 - Francis Sullivan
@@ -124,19 +124,6 @@ files:
 - lib/boxcars/boxcar/google_search.rb
 - lib/boxcars/boxcar/sql.rb
 - lib/boxcars/boxcar/swagger.rb
-- lib/boxcars/boxcar/vector_search.rb
-- lib/boxcars/boxcar/vector_store.rb
-- lib/boxcars/boxcar/vector_stores/document.rb
-- lib/boxcars/boxcar/vector_stores/embed_via_open_ai.rb
-- lib/boxcars/boxcar/vector_stores/embed_via_tensorflow.rb
-- lib/boxcars/boxcar/vector_stores/hnswlib/build_vector_store.rb
-- lib/boxcars/boxcar/vector_stores/hnswlib/hnswlib_config.rb
-- lib/boxcars/boxcar/vector_stores/hnswlib/hnswlib_search.rb
-- lib/boxcars/boxcar/vector_stores/hnswlib/save_to_hnswlib.rb
-- lib/boxcars/boxcar/vector_stores/in_memory/add_documents.rb
-- lib/boxcars/boxcar/vector_stores/in_memory/search.rb
-- lib/boxcars/boxcar/vector_stores/similarity_search.rb
-- lib/boxcars/boxcar/vector_stores/split_text.rb
 - lib/boxcars/boxcar/wikipedia_search.rb
 - lib/boxcars/conversation.rb
 - lib/boxcars/conversation_prompt.rb
@@ -152,6 +139,19 @@ files:
 - lib/boxcars/train/train_action.rb
 - lib/boxcars/train/train_finish.rb
 - lib/boxcars/train/zero_shot.rb
+- lib/boxcars/vector_search.rb
+- lib/boxcars/vector_store.rb
+- lib/boxcars/vector_store/document.rb
+- lib/boxcars/vector_store/embed_via_open_ai.rb
+- lib/boxcars/vector_store/embed_via_tensorflow.rb
+- lib/boxcars/vector_store/hnswlib/build_vector_store.rb
+- lib/boxcars/vector_store/hnswlib/hnswlib_config.rb
+- lib/boxcars/vector_store/hnswlib/hnswlib_search.rb
+- lib/boxcars/vector_store/hnswlib/save_to_hnswlib.rb
+- lib/boxcars/vector_store/in_memory/add_documents.rb
+- lib/boxcars/vector_store/in_memory/search.rb
+- lib/boxcars/vector_store/similarity_search.rb
+- lib/boxcars/vector_store/split_text.rb
 - lib/boxcars/version.rb
 homepage: https://github.com/BoxcarsAI/boxcars
 licenses:

data/lib/boxcars/boxcar/vector_store.rb

@@ -1,34 +0,0 @@
-# frozen_string_literal: true
-
-# Boxcars is a framework for running a series of tools to get an answer to a question.
-module Boxcars
-  module VectorStore
-    module ClassMethods
-      VectorStoresError = Class.new(StandardError)
-
-      def call(*args, **kw_args)
-        new(*args, **kw_args).call
-      end
-    end
-
-    def self.included(base)
-      base.extend(ClassMethods)
-
-      class << base
-        private :new
-      end
-    end
-  end
-end
-
-require_relative "vector_stores/document"
-require_relative "vector_stores/embed_via_open_ai"
-require_relative "vector_stores/embed_via_tensorflow"
-require_relative "vector_stores/split_text"
-require_relative "vector_stores/similarity_search"
-require_relative "vector_stores/hnswlib/hnswlib_config"
-require_relative "vector_stores/hnswlib/save_to_hnswlib"
-require_relative "vector_stores/hnswlib/build_vector_store"
-require_relative "vector_stores/hnswlib/hnswlib_search"
-require_relative "vector_stores/in_memory/add_documents"
-require_relative "vector_stores/in_memory/search"