weaviate_record 0.0.3

data/lib/weaviate_record/queries/count.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module WeaviateRecord
+  module Queries
+    # This class contains functions to perform count operation on Weaviate Relations
+    module Count
+      # Return the count of records matching the given conditions or search filters.
+      #
+      # +:bm25+ will not work here because it is not supported in aggregation queries
+      # +:limit+ and +:offset+ does not work with aggregation queries too
+      #
+      # ==== Example:
+      #     Article.where(title: 'movie').count #=> 1
+      def count
+        query = to_query.slice(:class_name, :near_text, :near_vector, :near_object, :where)
+        query[:fields] = 'meta { count }'
+        @connection.client.query.aggs(**query).dig(0, 'meta', 'count')
+      end
+    end
+  end
+end

data/lib/weaviate_record/queries/limit.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module WeaviateRecord
+  module Queries
+    # This module contains functions to perform limit query
+    module Limit
+      # Limit the number of records to be fetched from the database
+      #
+      # ==== Example:
+      #  articles = Article.limit(5)
+      #  articles.size #=> 5
+      def limit(limit_value)
+        raise TypeError, 'Limit should be as integer' unless limit_value.to_i.to_s == limit_value.to_s
+
+        @limit = limit_value
+        @loaded = false
+        self
+      end
+    end
+  end
+end

data/lib/weaviate_record/queries/near_object.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module WeaviateRecord
+  module Queries
+    # This module includes methods to perform 'near object' queries.
+    module NearObject
+      # Performs a similarity search based on the given object.
+      # This method takes either id of the object or the object itself and returns the list of objects
+      # that are nearer to it in terms of vector distance. You can also limit the distance by passing
+      # the distance parameter.
+      #
+      # ==== Example:
+      #   Article.create(content: 'This is a movie about friendship, action and adventure')
+      #   # => #<Article:0x00000001052091e8 id: "983c0970-2c65-4c38-a93f-2ca9272d784b"... >
+      #   obj = Article.create(content: 'This is a review about a movie')
+      #   # => #<Article:0x00000001052091e8 id: "0476e426-7e7f-4010-bfad-20c57a65c5c7"... >
+      #
+      #   Article.near_object(obj)
+      #   # => [... #<Article:0x00000001052091e8 id: "983c0970-2c65-4c38-a93f-2ca9272d784b"... > ]
+      def near_object(object, distance: WeaviateRecord.config.similarity_search_threshold)
+        unless object.is_a?(WeaviateRecord::Base) || object.is_a?(String)
+          raise TypeError, "Invalid type #{object.class} for near object query"
+        end
+
+        raise TypeError, 'Invalid uuid' if object.is_a?(String) && !Constants::UUID_REGEX.match?(object)
+        raise TypeError, 'Invalid value for distance' unless distance.is_a?(Numeric)
+
+        @near_object = "{ id: #{(object.is_a?(WeaviateRecord::Base) ? object.id : object).inspect}, " \
+                       "distance: #{distance} }"
+        @loaded = false
+        self
+      end
+    end
+  end
+end

data/lib/weaviate_record/queries/near_text.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module WeaviateRecord
+  module Queries
+    # This module contains functions to perform near_text query (Context Based Search)
+    module NearText
+      # Perform a similarity search on Weaviate collection.
+      # This method also takes an optional distance parameter to specify the threshold of similarity.
+      # You can also pass multiple texts to search in the collection.
+      #
+      # ==== Example:
+      #   Article.create(content: 'This is a movie about friendship, action and adventure')
+      #   # => #<Article:0x00000001052091e8 id: "983c0970-2c65-4c38-a93f-2ca9272d784b"... >
+      #
+      #   Article.near_text('review about a movie')
+      #   # => [#<Article:0x00000001052091e8 id: "983c0970-2c65-4c38-a93f-2ca9272d784b"... >]
+      def near_text(*texts, distance: WeaviateRecord.config.similarity_search_threshold)
+        raise TypeError, 'invalid value for text' unless texts.all? { |text| text.is_a?(String) }
+        raise TypeError, 'Invalid value for distance' unless distance.is_a?(Numeric)
+
+        @near_text_options[:distance] = distance
+        @near_text_options[:concepts] += texts.map! { |text| text.gsub('"', "'") }
+        @loaded = false
+        self
+      end
+
+      private
+
+      def formatted_near_text_value
+        texts = @near_text_options[:concepts].map(&:inspect).join(', ')
+
+        "{ concepts: [#{texts}], distance: #{@near_text_options[:distance]} }"
+      end
+    end
+  end
+end

data/lib/weaviate_record/queries/near_vector.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module WeaviateRecord
+  module Queries
+    # This module provides method for near vector query
+    module NearVector
+      # Performs a similarity search based on the given vector.
+      # This method takes a vector (Array of float values) and
+      # returns the list of objects that are nearer to it in terms of vector distance.
+      # You can also limit the distance by passing the distance parameter.
+      #
+      # ==== Example:
+      #  Article.create(content: 'This is a movie about friendship, action and adventure')
+      #  # => #<Article:0x00000001052091e8 id: "983c0970-2c65-4c38-a93f-2ca9272d784b"... >
+      #
+      #  vector = Article.select(_additional: :vector).where(id: "983c0970-2c65-4c38-a93f-2ca9272d784b").vector
+      #  # => [-0.37226558, 0.10700592, -0.3906307, 0.1064298 ... ]
+      #
+      #  Article.near_vector(vector)
+      #  # => [... #<Article:0x00000001052091e8 id: "983c0970-2c65-4c38-a93f-2ca9272d784b"... > ]
+      def near_vector(vector, distance: WeaviateRecord.config.similarity_search_threshold)
+        raise TypeError, "Invalid type #{vector.class} for near vector query" unless vector.is_a?(Array)
+        raise TypeError, 'Invalid vector' unless vector.all? { |v| v.is_a?(Float) }
+        raise TypeError, 'Invalid value for distance' unless distance.is_a?(Numeric)
+
+        @near_vector = "{ vector: #{vector}, distance: #{distance} }"
+        @loaded = false
+        self
+      end
+    end
+  end
+end

data/lib/weaviate_record/queries/offset.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module WeaviateRecord
+  module Queries
+    # This module contains function to offset Weaviate records
+    module Offset
+      # Offset the number of records to be fetched from the database
+      #
+      # ==== Example:
+      #   Article.count #=> 10
+      #   articles = Article.offset(5)
+      #   articles.size #=> 5
+      def offset(offset_value)
+        raise TypeError, 'Offset should be an integer' unless offset_value.to_i.to_s == offset_value.to_s
+
+        @offset = offset_value
+        @loaded = false
+        self
+      end
+    end
+  end
+end

data/lib/weaviate_record/queries/order.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module WeaviateRecord
+  module Queries
+    # This module contains function to sort Weaviate records
+    module Order
+      # Sort the records based on the given attributes.
+      # You can pass multiple attributes to sort the records.
+      # This sorting specification will be ignored if you are performing keyword (bm25) search.
+      #
+      # ==== Example:
+      #    Article.order(:title)
+      #    # Sorts the records based on title in ascending order
+      #
+      #    Article.order(:title, created_at: :desc)
+      #    # Sorts the records based on title in ascending order and created_at in descending order
+      def order(*args, **kw_args)
+        raise ArgumentError, 'expected at least one argument' if args.empty? && kw_args.empty?
+
+        sorting_specifiers = combine_arguments(args, kw_args)
+        assign_sort_options(sorting_specifiers)
+        @loaded = false
+        self
+      end
+
+      private
+
+      def validate_attribute_and_order(attribute, sorting_order)
+        unless attribute.is_a?(Symbol) || attribute.is_a?(String)
+          raise TypeError, 'Invalid type for sorting attribute, should be either type or symbol'
+        end
+
+        return if %i[asc desc].include? sorting_order
+
+        raise WeaviateRecord::Errors::SortingOptionError, 'Invalid sorting order'
+      end
+
+      def combine_arguments(args, kw_args)
+        [*args.map! { |attribute| convert_to_sorting_specifier(attribute) },
+         *kw_args.map { |attribute, sorting_order| convert_to_sorting_specifier(attribute, sorting_order) }]
+      end
+
+      def convert_to_sorting_specifier(attribute, sorting_order = :asc)
+        validate_attribute_and_order(attribute, sorting_order)
+        attribute = map_to_weaviate_attribute(attribute)
+
+        "{ path: [#{attribute.to_s.inspect}], order: #{sorting_order} }"
+      end
+
+      def map_to_weaviate_attribute(attribute)
+        return '_id' if attribute.to_s == 'id'
+        return attribute unless WeaviateRecord::Constants::SPECIAL_ATTRIBUTE_MAPPINGS.key?(attribute.to_s)
+
+        "_#{WeaviateRecord::Constants::SPECIAL_ATTRIBUTE_MAPPINGS[attribute.to_s]}"
+      end
+
+      def assign_sort_options(sorting_specifiers)
+        @sort_options = if @sort_options.nil?
+                          merge_sorting_specifiers(*sorting_specifiers)
+                        elsif @sort_options.start_with?('[')
+                          merge_sorting_specifiers(@sort_options[2...-2], *sorting_specifiers)
+                        else
+                          merge_sorting_specifiers(@sort_options, *sorting_specifiers)
+                        end
+      end
+
+      def merge_sorting_specifiers(*sorting_specifiers)
+        return sorting_specifiers[0] if sorting_specifiers.size == 1
+
+        "[ #{sorting_specifiers.join(', ')} ]"
+      end
+    end
+  end
+end

data/lib/weaviate_record/queries/select.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module WeaviateRecord
+  module Queries
+    # This module contains function to perform select query on Weaviate
+    module Select
+      # Select the attributes to be fetched from the database.
+      # You can also pass nested attributes to be fetched.
+      # Meta attributes that needs to be fetched should be passed aa a value for key '_additional'.
+      # In Weaviate, +id+ is also a meta attribute.
+      # If select is not called on a Weaviate query, by default
+      # it will fetch all normak attributes with id and timestamps.
+      #
+      # ==== Example:
+      #   Article.select(:content, :title)
+      #   Article.select(_additional: :vector)
+      #   Article.select( _additional: [:id, :created_at, :updated_at])
+      #   Article.select(_additional: { answer: :result })
+      #
+      #   Article.all #=> fetches id, content, title, created_at, updated_at
+      #
+      # There is one more special scenario where you can pass the graphql query directly.
+      # It will be used for summarization offered by summarizer module.
+      #
+      # ==== Example:
+      #   Article.select(_additional: 'summary(properties: ["content"]) { result }')
+      def select(*args)
+        args.each do |arg|
+          if arg.is_a? Hash
+            @select_options[:nested_attributes].merge! arg
+          else
+            @select_options[:attributes] << arg.to_s unless @select_options[:attributes].include?(arg.to_s)
+          end
+        end
+        @loaded = false
+        self
+      end
+
+      private
+
+      def combined_select_attributes
+        attributes = format_array_attribute(@select_options[:attributes])
+        return attributes if @select_options[:nested_attributes].empty?
+
+        "#{attributes} #{format_nested_attribute(@select_options[:nested_attributes])}"
+      end
+
+      def create_or_process_select_attributes(custom_selected, attributes)
+        if custom_selected
+          attributes.gsub(/(?<=\s)(#{WeaviateRecord::Constants::SPECIAL_ATTRIBUTE_MAPPINGS.keys.join('|')})(?=\s)/,
+                          WeaviateRecord::Constants::SPECIAL_ATTRIBUTE_MAPPINGS)
+        else
+          [
+            *WeaviateRecord::Schema.find_collection(@klass).attributes_list,
+            '_additional { id creationTimeUnix lastUpdateTimeUnix }'
+          ].join(' ')
+        end
+      end
+
+      def format_array_attribute(array)
+        array.map do |attribute|
+          case attribute
+          when String, Symbol then attribute.to_s
+          when Array then format_array_attribute(attribute)
+          when Hash then format_nested_attribute(attribute)
+          else raise TypeError
+          end
+        end.join(' ')
+      end
+
+      def format_nested_attribute(hash)
+        return_string = String.new
+        hash.each do |key, value|
+          return_string << key.to_s << ' { '
+          case value
+          when String, Symbol then return_string << value.to_s << ' } '
+          when Array then return_string << format_array_attribute(value) << ' } '
+          when Hash then return_string << format_nested_attribute(value) << ' } ' else raise TypeError
+          end
+        end
+        return_string.rstrip
+      end
+    end
+  end
+end

data/lib/weaviate_record/queries/where.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module WeaviateRecord
+  module Queries
+    # This module contains function to perform where query on Weaviate
+    module Where
+      # Perform a where query on the collection. You can pass a string or keyword arguments as a query.
+      # It follows the same syntax as ActiveRecord where query. Chaining of where queries is also supported.
+      #
+      # ==== Example:
+      #  Article.where('title = ?', 'Hello World')
+      #  Article.where(title: 'Hello World')
+      #  Article.where('title = ? AND content = ?', 'Hello World', 'This is a content')
+      #  Article.where(title: 'Hello World').where(content: 'This is a content')
+      def where(query = '', *values, **kw_args)
+        validate_arguments(query, values, kw_args)
+        keyword_query = process_keyword_conditions(kw_args)
+        string_query = process_string_conditions(query, *values)
+        combined_query = combine_queries(keyword_query, string_query)
+        @where_query = @where_query ? create_logical_condition(@where_query, 'And', combined_query) : combined_query
+        @loaded = false
+        self
+      end
+
+      # :enddoc:
+
+      def self.to_ruby_hash(string_condition)
+        pattern = /(?<=\s)\w+:|(?<=operator:\s)\w+/
+        keys_and_operator = string_condition.scan(pattern).uniq
+        json_equivalent = keys_and_operator.map { |i| i.end_with?(':') ? "#{i[0...-1].inspect}:" : i.inspect }
+        JSON.parse string_condition.gsub(pattern, keys_and_operator.zip(json_equivalent).to_h)
+      rescue StandardError
+        raise WeaviateRecord::Errors::WhereQueryConversionError, 'invalid where query format'
+      end
+
+      private
+
+      def validate_arguments(query, values, kw_args)
+        if values.empty? && kw_args.empty?
+          raise WeaviateRecord::Errors::InvalidWhereQueryError, 'invalid argument for where query'
+        end
+
+        return unless values.size != query.count('?')
+
+        raise WeaviateRecord::Errors::InvalidWhereQueryError, 'invalid number of arguments'
+      end
+
+      def process_keyword_conditions(hash)
+        return nil if hash.empty?
+
+        conditions = hash.each_pair.map do |key, value|
+          create_query_condition([key.to_s, value.is_a?(Array) ? 'CONTAINS_ANY' : '=', value])
+        end
+        conditions.inject { |acc, condition| create_logical_condition(acc, 'AND', condition) }
+      end
+
+      def process_string_conditions(query, *values)
+        return nil unless query.present? && values.present?
+
+        logical_operator_match = /\s+(AND|OR)\s+/i.match(query)
+        return create_query_condition_from_string(query, values) unless logical_operator_match
+
+        pre_condition = create_query_condition_from_string(logical_operator_match.pre_match, values)
+        post_condition = process_string_conditions(logical_operator_match.post_match, *values)
+
+        create_logical_condition(pre_condition, logical_operator_match[1], post_condition)
+      end
+
+      def create_query_condition_from_string(condition, values)
+        equation = condition.split
+        raise WeaviateRecord::Errors::InvalidWhereQueryError, 'unable to process the query' unless equation.size == 3
+        raise WeaviateRecord::Errors::InvalidWhereQueryError, 'insufficient values for formatting' if values.empty?
+
+        equation[-1] = values.shift
+        create_query_condition(equation)
+      end
+
+      def combine_queries(first_query, second_query)
+        if first_query.present? && second_query.present?
+          create_logical_condition(first_query, 'And', second_query)
+        else
+          first_query.presence || second_query
+        end
+      end
+
+      def create_query_condition(equation)
+        return null_condition(equation[0]) if equation[2].nil?
+
+        handle_timestamps_condition(equation)
+        "{ path: [\"#{equation[0]}\"], " \
+          "operator: #{map_operator(equation[1])}, " \
+          "#{map_value_type(equation[2])}: #{equation[2].inspect} }"
+      end
+
+      def handle_timestamps_condition(equation_array)
+        return nil unless equation_array[0] == 'created_at' || equation_array[0] == 'updated_at'
+
+        equation_array[0] = "_#{WeaviateRecord::Constants::SPECIAL_ATTRIBUTE_MAPPINGS[equation_array[0]]}"
+        equation_array[2] = equation_array[2].to_datetime.strftime('%Q')
+      end
+
+      def null_condition(attribute)
+        "{ path: [\"#{attribute}\"], operator: IsNull, valueBoolean: true }"
+      end
+
+      def create_logical_condition(pre_condition, operator, post_condition)
+        "{ operator: #{operator.capitalize}, " \
+          "operands: [#{pre_condition}, #{post_condition}] }"
+      end
+
+      def map_operator(operator)
+        WeaviateRecord::Constants::OPERATOR_MAPPING_HASH.fetch(operator) do
+          raise WeaviateRecord::Errors::InvalidOperatorError, "Invalid conditional operator #{operator}"
+        end
+      end
+
+      def map_value_type(value)
+        WeaviateRecord::Constants::TYPE_MAPPING_HASH.fetch(value.class) do |klass|
+          raise WeaviateRecord::Errors::InvalidValueTypeError, "Invalid value type #{klass} for comparison"
+        end
+      end
+    end
+  end
+end

data/lib/weaviate_record/relation/query_builder.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module WeaviateRecord
+  class Relation
+    # This module contains methods which helps to build query for Weaviate
+    module QueryBuilder
+      # This will return the query that will be sent to Weaviate. More like +to_sql+ in ActiveRecord.
+      #
+      # ==== Example:
+      #   Article.select(:title, :content).near_text('friendship movie').limit(5).offset(2).to_query
+      # Returns:
+      #   {:class_name=>"Article",
+      #    :limit=>"5",
+      #    :offset=>"2",
+      #    :fields=>"title content",
+      #    :near_text=>"{ concepts: [\"friendship movie\"], distance: 0.55 }"}
+      def to_query
+        query_params = basic_params
+        fill_up_keyword_search_param(query_params)
+        fill_up_similarity_search_param(query_params)
+        fill_up_conditions_param(query_params)
+        fill_up_sort_param(query_params)
+        fill_up_question_param(query_params)
+
+        query_params
+      end
+
+      private
+
+      def basic_params
+        { class_name: @klass.to_s, limit: @limit.to_s, offset: @offset.to_s,
+          fields: combined_select_attributes }
+      end
+
+      def fill_up_keyword_search_param(query_params)
+        query_params[:bm25] = @keyword_search if @keyword_search.present?
+      end
+
+      def fill_up_similarity_search_param(query_params)
+        query_params[:near_text] = formatted_near_text_value unless @near_text_options[:concepts].empty?
+        query_params[:near_vector] = @near_vector if @near_vector
+        query_params[:near_object] = @near_object if @near_object
+      end
+
+      def fill_up_conditions_param(query_params)
+        query_params[:where] = @where_query if @where_query
+      end
+
+      def fill_up_sort_param(query_params)
+        # Weaviate doesn't support sorting with bm25 search at the time of writing this code.
+        query_params[:sort] = @sort_options if @keyword_search.blank? && @sort_options
+      end
+
+      def fill_up_question_param(query_params)
+        query_params[:ask] = @ask if @ask
+      end
+    end
+  end
+end

data/lib/weaviate_record/relation.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+
+module WeaviateRecord
+  # This class is used to build weaviate queries
+  class Relation
+    extend Forwardable
+    include Enumerable
+    include Queries::Bm25
+    include Queries::Count
+    include Queries::Limit
+    include Queries::NearText
+    include Queries::NearVector
+    include Queries::NearObject
+    include Queries::Offset
+    include Queries::Order
+    include Queries::Select
+    include Queries::Where
+    include Queries::Ask
+    include QueryBuilder
+
+    def_delegators(:records, :empty?, :present?, :[], :first, :last)
+
+    # :stopdoc:
+    def initialize(klass)
+      @select_options = { attributes: [], nested_attributes: {} }
+      @near_text_options = { concepts: [], distance: WeaviateRecord.config.similarity_search_threshold }
+      @limit = ENV['QUERY_DEFAULTS_LIMIT'] || 25
+      @offset = 0
+      @klass = klass
+      @records = []
+      @loaded = false
+      @connection = WeaviateRecord::Connection.new(@klass)
+    end
+    # :startdoc:
+
+    # To enumerate over each record in the Weaviate relation
+    def each(&block)
+      records.each(&block)
+    end
+
+    # Gets all the records from Weaviate matching the given conditions or search filters given in the query.
+    # This will return an array of WeaviateRecord objects.
+    def all
+      records
+    rescue StandardError => e
+      e
+    end
+
+    # Deletes all the records from Weaviate matching the given conditions or search filters given in the query.
+    # This will return the result of batch delete operation given by Weaviate.
+    #
+    # ==== Example:
+    #   Article.where(title: nil).destroy_all
+    #   # => {"failed"=>0, "limit"=>10000, "matches"=>3, "objects"=>nil, "successful"=>3}
+    def destroy_all
+      unless @where_query
+        raise WeaviateRecord::Errors::MissingWhereCondition, 'must specifiy atleast one where condition'
+      end
+
+      response = @connection.delete_where(Queries::Where.to_ruby_hash(@where_query))
+      return response['results'] if response.is_a?(Hash) && response.key?('results')
+
+      raise WeaviateRecord::Errors::ServerError,
+            response == '' ? 'Unauthorized' : response.dig('error', 'message').presence
+    end
+
+    alias inspect all
+    alias to_a all
+
+    private
+
+    def records
+      return @records if @loaded
+
+      query = to_query
+      custom_selected = query[:fields].present?
+      query[:fields] = create_or_process_select_attributes(custom_selected, query[:fields])
+      result = @connection.client.query.get(**query)
+      @loaded = true
+      @records = result.map { |record| @klass.new(custom_selected: custom_selected, **record) }
+      @records
+    end
+  end
+end