speaky 0.1.6 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3dea3c0900c9679d76d464ded63bc6c3d8cfad6d03da8661ad19b1013779b255
-  data.tar.gz: 62858d2b9e755c8c56a18b34d0a5651c8dcdb5687b15b1294d1a3c48986a4176
+  metadata.gz: d65d925dd1dc599e493c9ebb83f4e7eff915aa5ffffbc2daf4248bb83d1801fd
+  data.tar.gz: 6714917b885cc8c34992b75ff3688a60076568a341d6ccee577f8b5959946da8
 SHA512:
-  metadata.gz: 304614f14107c1a2effe210d1d00f599fab93ce6f1a086d09f398b0f7b42b0b129b95d585f3a6fccb36b9d3a0cee797342365634392ee15430ade4e3ccbf2aca
-  data.tar.gz: '01229a40b984e5165e8ed669df1d94c8923ddc1f1b42a02ff3557215b6301531389c44e1a744092785fb1312842b498c7160c53888a7b5b640a587d677986e8c'
+  metadata.gz: 6a9c1ca13995cf513c4600725fb9a5f3ed6a2749a9d020ecd204dacf07e528356dd9164cb64d9ff70b744fae5938efe274825186ea374cec9599aa51bed07187
+  data.tar.gz: 3ffc25af7d5884ab66720ef78fc03b6afe10892b0a3296be83a53e150654ebf09410bdae9eb33e164cc2ec485add2771aa84f72623c605892f4761c32ec8a0cc

data/lib/speaky/concern.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'rails'
 require 'active_support'
 
 module Speaky

data/lib/speaky/vectorstore_base.rb

@@ -25,7 +25,7 @@ module Speaky
     end
 
     # Query the vectorstore with a question.
-    # Returns a string.
+    # Returns an array of strings.
     def query(question)
       raise NotImplementedError
     end

data/lib/speaky/vectorstore_faiss.rb

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
+require 'faiss'
+
 module Speaky
   class VectorstoreFaiss < VectorstoreBase
     def initialize(config)
-      raise 'This class is not implemented yet.' # TEMP
-
       @config = config
 
       # check if the index path is set
@@ -15,11 +15,37 @@ module Speaky
         @index = Faiss::Index.load(@config[:index_path])
       else
         # create a new index
-        @index = Faiss::IndexFlatL2.new(768)
+        index = Faiss::IndexFlatL2.new(1536)
+        @index = Faiss::IndexIDMap.new(index)
         @index.save(@config[:index_path])
       end
     end
 
-    # TODO: Implement the other methods
+    def add(id, data)
+      embeddings = Speaky.llm.embed(data)
+
+      @index.add_with_ids([embeddings], [string_id_to_unique_int_id(id)])
+
+      true
+    end
+
+    def remove(id)
+      # remove is not supported by Faiss
+      true
+    end
+
+    def query(question)
+      # TODO: implement query
+
+      []
+    end
+
+    private
+
+    # This method is used to convert a string ID to a unique integer ID
+    # that can be used by the Qdrant API.
+    def string_id_to_unique_int_id(string_id)
+      string_id.to_s.hash.abs
+    end
   end
 end

data/lib/speaky/vectorstore_qdrant.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'rails'
 require 'qdrant'
 
 module Speaky
@@ -29,7 +30,7 @@ module Speaky
         collection_name: @config[:collection_name],
         points: [
           {
-            id: id,
+            id: string_id_to_unique_int_id(id),
             vector: embeddings,
             payload: {
               content: data
@@ -50,7 +51,7 @@ module Speaky
     def remove(id)
       points_delete = @client.points.delete(
         collection_name: @config[:collection_name],
-        points: [id],
+        points: [string_id_to_unique_int_id(id)],
       )
 
       if !points_delete || points_delete.dig('status') != 'ok'
@@ -66,7 +67,7 @@ module Speaky
 
       points_search = @client.points.search(
         collection_name: @config[:collection_name],
-        limit: 1,
+        limit: 5,
         vector: embeddings,
         with_payload: true,
         with_vector: false
@@ -77,7 +78,7 @@ module Speaky
         raise 'Failed to search vectors'
       end
 
-      points_search.dig('result').first.dig('payload', 'content')
+      points_search.dig('result').map { |r| r.dig('payload', 'content') }
     end
 
     def reset
@@ -117,5 +118,11 @@ module Speaky
 
       true
     end
+
+    # This method is used to convert a string ID to a unique integer ID
+    # that can be used by the Qdrant API.
+    def string_id_to_unique_int_id(string_id)
+      string_id.to_s.hash.abs
+    end
   end
 end

data/lib/speaky/version.rb

@@ -1,3 +1,3 @@
 module Speaky
-  VERSION = '0.1.6'
+  VERSION = '0.1.8'
 end

data/lib/speaky.rb

@@ -87,7 +87,7 @@ module Speaky
       template ||= default_template
 
       # load context
-      context = vectorstore.query(question)
+      context = vectorstore.query(question).join("\n")
 
       # generate prompt
       prompt = template.gsub("{{context}}", context).gsub("{{question}}", question)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: speaky
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.8
 platform: ruby
 authors:
 - Gregorio Galante