typesense 0.2.0 → 0.4.0

data/examples/overrides.rb

@@ -3,30 +3,9 @@
 ##
 # These examples walk you through operations specifically related to overrides
 # This is a Typesense Premium feature (see: https://typesense.org/premium)
+# Be sure to add `--license-key=<>` as a parameter, when starting a Typesense Premium server
 
-require_relative '../lib/typesense'
-require 'awesome_print'
-
-AwesomePrint.defaults = {
-  indent: -2
-}
-
-##
-# Setup
-#
-# Start the master
-#   $ docker run -p 8108:8108  -it -v/tmp/typesense-data-master/:/data -it typesense/typesense:0.8.0-rc1 --data-dir /data --api-key=abcd --listen-port 8108 --license-key=<>
-
-##
-# Create a client
-typesense = Typesense::Client.new(
-  master_node: {
-    host: 'localhost',
-    port: 8108,
-    protocol: 'http',
-    api_key: 'abcd'
-  }
-)
+require_relative './client_initialization'
 
 ##
 # Create a collection
@@ -50,31 +29,31 @@ schema = {
   'default_sorting_field' => 'num_employees'
 }
 
-typesense.collections.create(schema)
+@typesense.collections.create(schema)
 
 # Let's create a couple documents for us to use in our search examples
-typesense.collections['companies'].documents.create(
+@typesense.collections['companies'].documents.create(
   'id' => '124',
   'company_name' => 'Stark Industries',
   'num_employees' => 5215,
   'country' => 'USA'
 )
 
-typesense.collections['companies'].documents.create(
+@typesense.collections['companies'].documents.create(
   'id' => '127',
   'company_name' => 'Stark Corp',
   'num_employees' => 1031,
   'country' => 'USA'
 )
 
-typesense.collections['companies'].documents.create(
+@typesense.collections['companies'].documents.create(
   'id' => '125',
   'company_name' => 'Acme Corp',
   'num_employees' => 1002,
   'country' => 'France'
 )
 
-typesense.collections['companies'].documents.create(
+@typesense.collections['companies'].documents.create(
   'id' => '126',
   'company_name' => 'Doofenshmirtz Inc',
   'num_employees' => 2,
@@ -84,7 +63,7 @@ typesense.collections['companies'].documents.create(
 ##
 # Create overrides
 
-typesense.collections['companies'].overrides.create(
+@typesense.collections['companies'].overrides.create(
   "id": 'promote-doofenshmirtz',
   "rule": {
     "query": 'doofen',
@@ -92,7 +71,7 @@ typesense.collections['companies'].overrides.create(
   },
   "includes": [{ 'id' => '126', 'position' => 1 }]
 )
-typesense.collections['companies'].overrides.create(
+@typesense.collections['companies'].overrides.create(
   "id": 'promote-acme',
   "rule": {
     "query": 'stark',
@@ -103,19 +82,19 @@ typesense.collections['companies'].overrides.create(
 
 ##
 # Search for documents
-results = typesense.collections['companies'].documents.search(
+results = @typesense.collections['companies'].documents.search(
   'q' => 'doofen',
   'query_by' => 'company_name'
 )
 ap results
 
-results = typesense.collections['companies'].documents.search(
+results = @typesense.collections['companies'].documents.search(
   'q' => 'stark',
   'query_by' => 'company_name'
 )
 ap results
 
-results = typesense.collections['companies'].documents.search(
+results = @typesense.collections['companies'].documents.search(
   'q' => 'Inc',
   'query_by' => 'company_name',
   'filter_by' => 'num_employees:<100',
@@ -126,4 +105,4 @@ ap results
 ##
 # Cleanup
 # Drop the collection
-typesense.collections['companies'].delete
+@typesense.collections['companies'].delete

data/examples/search.rb

@@ -3,41 +3,7 @@
 ##
 # These examples walk you through operations specifically related to search
 
-require_relative '../lib/typesense'
-require 'awesome_print'
-
-AwesomePrint.defaults = {
-  indent: -2
-}
-
-##
-# Setup
-#
-# Start the master
-#   $ docker run -p 8108:8108  -it -v/tmp/typesense-data-master/:/data -it typesense/typesense:0.8.0-rc1 --data-dir /data --api-key=abcd --listen-port 8108
-#
-# Start the read replica
-#   $ docker run -p 8109:8109  -it -v/tmp/typesense-data-read-replica-1/:/data -it typesense/typesense:0.8.0-rc1 --data-dir /data --api-key=wxyz --listen-port 8109 --master http://localhost:8108
-
-##
-# Create a client
-typesense = Typesense::Client.new(
-  master_node: {
-    host: 'localhost',
-    port: 8108,
-    protocol: 'http',
-    api_key: 'abcd'
-  },
-  read_replica_nodes: [
-    {
-      host: 'localhost',
-      port: 8109,
-      protocol: 'http',
-      api_key: 'wxyz'
-    }
-  ],
-  timeout_seconds: 10
-)
+require_relative './client_initialization'
 
 ##
 # Create a collection
@@ -61,31 +27,38 @@ schema = {
   'default_sorting_field' => 'num_employees'
 }
 
-typesense.collections.create(schema)
+# Delete the collection if it already exists
+begin
+  @typesense.collections['companies'].delete
+rescue Typesense::Error::ObjectNotFound
+end
+
+# Now create the collection
+@typesense.collections.create(schema)
 
 # Let's create a couple documents for us to use in our search examples
-typesense.collections['companies'].documents.create(
+@typesense.collections['companies'].documents.create(
   'id' => '124',
   'company_name' => 'Stark Industries',
   'num_employees' => 5215,
   'country' => 'USA'
 )
 
-typesense.collections['companies'].documents.create(
+@typesense.collections['companies'].documents.create(
   'id' => '127',
   'company_name' => 'Stark Corp',
   'num_employees' => 1031,
   'country' => 'USA'
 )
 
-typesense.collections['companies'].documents.create(
+@typesense.collections['companies'].documents.create(
   'id' => '125',
   'company_name' => 'Acme Corp',
   'num_employees' => 1002,
   'country' => 'France'
 )
 
-typesense.collections['companies'].documents.create(
+@typesense.collections['companies'].documents.create(
   'id' => '126',
   'company_name' => 'Doofenshmirtz Inc',
   'num_employees' => 2,
@@ -94,7 +67,7 @@ typesense.collections['companies'].documents.create(
 
 ##
 # Search for documents
-results = typesense.collections['companies'].documents.search(
+results = @typesense.collections['companies'].documents.search(
   'q' => 'Stark',
   'query_by' => 'company_name'
 )
@@ -133,7 +106,7 @@ ap results
 
 ##
 # Search for more documents
-results = typesense.collections['companies'].documents.search(
+results = @typesense.collections['companies'].documents.search(
   'q' => 'Inc',
   'query_by' => 'company_name',
   'filter_by' => 'num_employees:<100',
@@ -163,7 +136,7 @@ ap results
 
 ##
 # Search for more documents
-results = typesense.collections['companies'].documents.search(
+results = @typesense.collections['companies'].documents.search(
   'q' => 'Non-existent',
   'query_by' => 'company_name'
 )
@@ -179,4 +152,4 @@ ap results
 ##
 # Cleanup
 # Drop the collection
-typesense.collections['companies'].delete
+@typesense.collections['companies'].delete

data/lib/typesense/alias.rb

@@ -2,17 +2,17 @@
 
 module Typesense
   class Alias
-    def initialize(configuration, name)
-      @configuration = configuration
-      @name          = name
+    def initialize(name, api_call)
+      @name     = name
+      @api_call = api_call
     end
 
     def retrieve
-      ApiCall.new(@configuration).get(endpoint_path)
+      @api_call.get(endpoint_path)
     end
 
     def delete
-      ApiCall.new(@configuration).delete(endpoint_path)
+      @api_call.delete(endpoint_path)
     end
 
     private

data/lib/typesense/aliases.rb

@@ -4,21 +4,21 @@ module Typesense
   class Aliases
     RESOURCE_PATH = '/aliases'
 
-    def initialize(configuration)
-      @configuration = configuration
+    def initialize(api_call)
+      @api_call = api_call
       @aliases = {}
     end
 
     def upsert(alias_name, mapping)
-      ApiCall.new(@configuration).put(endpoint_path(alias_name), mapping)
+      @api_call.put(endpoint_path(alias_name), mapping)
     end
 
     def retrieve
-      ApiCall.new(@configuration).get(RESOURCE_PATH)
+      @api_call.get(RESOURCE_PATH)
     end
 
     def [](alias_name)
-      @aliases[alias_name] ||= Alias.new(@configuration, alias_name)
+      @aliases[alias_name] ||= Alias.new(alias_name, @api_call)
     end
 
     private

data/lib/typesense/api_call.rb

@@ -10,116 +10,223 @@ module Typesense
 
     def initialize(configuration)
       @configuration = configuration
+
+      @api_key = @configuration.api_key
+      @nodes = @configuration.nodes.dup # Make a copy, since we'll be adding additional metadata to the nodes
+      @nearest_node = @configuration.nearest_node.dup
+      @connection_timeout_seconds = @configuration.connection_timeout_seconds
+      @healthcheck_interval_seconds = @configuration.healthcheck_interval_seconds
+      @num_retries_per_request = @configuration.num_retries
+      @retry_interval_seconds = @configuration.retry_interval_seconds
+
+      @logger = @configuration.logger
+
+      initialize_metadata_for_nodes
+      @current_node_index = -1
     end
 
     def post(endpoint, parameters = {})
-      perform_with_error_handling(:do_not_use_read_replicas) do
-        self.class.post(uri_for(endpoint),
-                        default_options.merge(
-                          body: parameters.to_json,
-                          headers: default_headers.merge('Content-Type' => 'application/json')
-                        ))
-      end.parsed_response
+      headers, body = extract_headers_and_body_from(parameters)
+
+      perform_request :post,
+                      endpoint,
+                      body: body,
+                      headers: default_headers.merge(headers)
     end
 
     def put(endpoint, parameters = {})
-      perform_with_error_handling(:do_not_use_read_replicas) do
-        self.class.put(uri_for(endpoint),
-                       default_options.merge(
-                         body: parameters.to_json,
-                         headers: default_headers.merge('Content-Type' => 'application/json')
-                       ))
-      end.parsed_response
+      headers, body = extract_headers_and_body_from(parameters)
+
+      perform_request :put,
+                      endpoint,
+                      body: body,
+                      headers: default_headers.merge(headers)
     end
 
     def get(endpoint, parameters = {})
-      get_unparsed_response(endpoint, parameters).parsed_response
-    end
+      headers, query = extract_headers_and_query_from(parameters)
 
-    def get_unparsed_response(endpoint, parameters = {})
-      perform_with_error_handling(:use_read_replicas) do |node, node_index|
-        self.class.get(uri_for(endpoint, node, node_index),
-                       default_options.merge(
-                         query: parameters,
-                         headers: default_headers
-                       ))
-      end
+      perform_request :get,
+                      endpoint,
+                      query: query,
+                      headers: default_headers.merge(headers)
     end
 
     def delete(endpoint, parameters = {})
-      perform_with_error_handling(:do_not_use_read_replicas) do
-        self.class.delete(uri_for(endpoint),
-                          default_options.merge(
-                            query: parameters,
-                            headers: default_headers
-                          ))
-      end.parsed_response
+      headers, query = extract_headers_and_query_from(parameters)
+
+      perform_request :delete,
+                      endpoint,
+                      query: query,
+                      headers: default_headers.merge(headers)
+    end
+
+    def perform_request(method, endpoint, options = {})
+      @configuration.validate!
+      last_exception = nil
+      @logger.debug "Performing #{method.to_s.upcase} request: #{endpoint}"
+      (1..(@num_retries_per_request + 1)).each do |num_tries|
+        node = next_node
+
+        @logger.debug "Attempting #{method.to_s.upcase} request Try ##{num_tries} to Node #{node[:index]}"
+
+        begin
+          response_object = self.class.send(method,
+                                            uri_for(endpoint, node),
+                                            default_options.merge(options))
+          response_code = response_object.response.code.to_i
+          set_node_healthcheck(node, is_healthy: true) if response_code >= 1 && response_code <= 499
+
+          @logger.debug "Request to Node #{node[:index]} was successfully made (at the network layer). Response Code was #{response_code}."
+
+          # If response is 2xx return the object, else raise the response as an exception
+          return response_object.parsed_response if response_object.response.code_type <= Net::HTTPSuccess # 2xx
+
+          raise custom_exception_klass_for(response_object.response), response_object.parsed_response['message'] || 'Error message not available'
+        rescue Net::ReadTimeout, Net::OpenTimeout,
+               EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError,
+               Errno::EINVAL, Errno::ENETDOWN, Errno::ENETUNREACH, Errno::ENETRESET, Errno::ECONNABORTED, Errno::ECONNRESET,
+               Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::EHOSTDOWN, Errno::EHOSTUNREACH,
+               Timeout::Error, HTTParty::ResponseError, Typesense::Error::ServerError, Typesense::Error::HTTPStatus0Error => e
+          # Rescue network layer exceptions and HTTP 5xx errors, so the loop can continue.
+          # Using loops for retries instead of rescue...retry to maintain consistency with client libraries in
+          #   other languages that might not support the same construct.
+          set_node_healthcheck(node, is_healthy: false)
+          last_exception = e
+          @logger.warn "Request to Node #{node[:index]} failed due to \"#{e.class}: #{e.message}\""
+          @logger.warn "Sleeping for #{@retry_interval_seconds}s and then retrying request..."
+          sleep @retry_interval_seconds
+        end
+      end
+      @logger.debug "No retries left. Raising last error \"#{last_exception.class}: #{last_exception.message}\"..."
+      raise last_exception
     end
 
     private
 
-    def uri_for(endpoint, node = :master, node_index = 0)
-      if node == :read_replica
-        "#{@configuration.read_replica_nodes[node_index][:protocol]}://#{@configuration.read_replica_nodes[node_index][:host]}:#{@configuration.read_replica_nodes[node_index][:port]}#{endpoint}"
+    def extract_headers_and_body_from(parameters)
+      if json_request?(parameters)
+        headers = { 'Content-Type' => 'application/json' }
+        body = sanitize_parameters(parameters).to_json
       else
-        "#{@configuration.master_node[:protocol]}://#{@configuration.master_node[:host]}:#{@configuration.master_node[:port]}#{endpoint}"
+        headers = {}
+        body = parameters[:body]
       end
+      [headers, body]
     end
 
-    def perform_with_error_handling(use_read_replicas = :do_not_use_read_replicas)
-      @configuration.validate!
+    def extract_headers_and_query_from(parameters)
+      if json_request?(parameters)
+        headers = { 'Content-Type' => 'application/json' }
+        query = sanitize_parameters(parameters)
+      else
+        headers = {}
+        query = parameters[:query]
+      end
+      [headers, query]
+    end
+
+    def json_request?(parameters)
+      parameters[:as_json].nil? ? true : parameters[:as_json]
+    end
+
+    def sanitize_parameters(parameters)
+      sanitized_parameters = parameters.dup
+      sanitized_parameters.delete(:as_json)
+      sanitized_parameters.delete(:body)
+      sanitized_parameters.delete(:query)
 
-      node       = :master
-      node_index = -1
-
-      begin
-        response_object = yield node, node_index
-
-        return response_object if response_object.response.code_type <= Net::HTTPSuccess # 2xx
-
-        error_klass = if response_object.response.code_type <= Net::HTTPBadRequest # 400
-                        Error::RequestMalformed
-                      elsif response_object.response.code_type <= Net::HTTPUnauthorized # 401
-                        Error::RequestUnauthorized
-                      elsif response_object.response.code_type <= Net::HTTPNotFound # 404
-                        Error::ObjectNotFound
-                      elsif response_object.response.code_type <= Net::HTTPConflict # 409
-                        Error::ObjectAlreadyExists
-                      elsif response_object.response.code_type <= Net::HTTPUnprocessableEntity # 422
-                        Error::ObjectUnprocessable
-                      elsif response_object.response.code_type <= Net::HTTPServerError # 5xx
-                        Error::ServerError
-                      else
-                        Error
-                      end
-
-        raise error_klass, response_object.parsed_response['message']
-      rescue Net::ReadTimeout, Net::OpenTimeout,
-             EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError,
-             Errno::EINVAL, Errno::ENETDOWN, Errno::ENETUNREACH, Errno::ENETRESET, Errno::ECONNABORTED, Errno::ECONNRESET,
-             Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::EHOSTDOWN, Errno::EHOSTUNREACH,
-             Timeout::Error, Error::ServerError, HTTParty::ResponseError
-        if ([:use_read_replicas, true].include? use_read_replicas) &&
-           !@configuration.read_replica_nodes.nil?
-          node = :read_replica
-          node_index += 1
-
-          retry unless @configuration.read_replica_nodes[node_index].nil?
+      sanitized_parameters
+    end
+
+    def uri_for(endpoint, node)
+      "#{node[:protocol]}://#{node[:host]}:#{node[:port]}#{endpoint}"
+    end
+
+    ## Attempts to find the next healthy node, looping through the list of nodes once.
+    #   But if no healthy nodes are found, it will just return the next node, even if it's unhealthy
+    #     so we can try the request for good measure, in case that node has become healthy since
+    def next_node
+      # Check if nearest_node is set and is healthy, if so return it
+      unless @nearest_node.nil?
+        @logger.debug "Nodes health: Node #{@nearest_node[:index]} is #{@nearest_node[:is_healthy] == true ? 'Healthy' : 'Unhealthy'}"
+        if @nearest_node[:is_healthy] == true || node_due_for_healthcheck?(@nearest_node)
+          @logger.debug "Updated current node to Node #{@nearest_node[:index]}"
+          return @nearest_node
+        end
+        @logger.debug 'Falling back to individual nodes'
+      end
+
+      # Fallback to nodes as usual
+      @logger.debug "Nodes health: #{@nodes.each_with_index.map { |node, i| "Node #{i} is #{node[:is_healthy] == true ? 'Healthy' : 'Unhealthy'}" }.join(' || ')}"
+      candidate_node = nil
+      (0..@nodes.length).each do |_i|
+        @current_node_index = (@current_node_index + 1) % @nodes.length
+        candidate_node = @nodes[@current_node_index]
+        if candidate_node[:is_healthy] == true || node_due_for_healthcheck?(candidate_node)
+          @logger.debug "Updated current node to Node #{candidate_node[:index]}"
+          return candidate_node
         end
+      end
+
+      # None of the nodes are marked healthy, but some of them could have become healthy since last health check.
+      # So we will just return the next node.
+      @logger.debug "No healthy nodes were found. Returning the next node, Node #{candidate_node[:index]}"
+      candidate_node
+    end
 
-        raise
+    def node_due_for_healthcheck?(node)
+      is_due_for_check = Time.now.to_i - node[:last_access_timestamp] > @healthcheck_interval_seconds
+      @logger.debug "Node #{node[:index]} has exceeded healthcheck_interval_seconds of #{@healthcheck_interval_seconds}. Adding it back into rotation." if is_due_for_check
+      is_due_for_check
+    end
+
+    def initialize_metadata_for_nodes
+      unless @nearest_node.nil?
+        @nearest_node[:index] = 'nearest_node'
+        set_node_healthcheck(@nearest_node, is_healthy: true)
+      end
+      @nodes.each_with_index do |node, index|
+        node[:index] = index
+        set_node_healthcheck(node, is_healthy: true)
+      end
+    end
+
+    def set_node_healthcheck(node, is_healthy:)
+      node[:is_healthy] = is_healthy
+      node[:last_access_timestamp] = Time.now.to_i
+    end
+
+    def custom_exception_klass_for(response)
+      response_code_type = response.code_type
+      if response_code_type <= Net::HTTPBadRequest # 400
+        Typesense::Error::RequestMalformed
+      elsif response_code_type <= Net::HTTPUnauthorized # 401
+        Typesense::Error::RequestUnauthorized
+      elsif response_code_type <= Net::HTTPNotFound # 404
+        Typesense::Error::ObjectNotFound
+      elsif response_code_type <= Net::HTTPConflict # 409
+        Typesense::Error::ObjectAlreadyExists
+      elsif response_code_type <= Net::HTTPUnprocessableEntity # 422
+        Typesense::Error::ObjectUnprocessable
+      elsif response_code_type <= Net::HTTPServerError # 5xx
+        Typesense::Error::ServerError
+      elsif response.code.to_i.zero?
+        Typesense::Error::HTTPStatus0Error
+      else
+        Typesense::Error::HTTPError
       end
     end
 
     def default_options
       {
-        timeout: @configuration.timeout_seconds
+        timeout: @connection_timeout_seconds
       }
     end
 
     def default_headers
       {
-        API_KEY_HEADER_NAME.to_s => @configuration.master_node[:api_key]
+        API_KEY_HEADER_NAME.to_s => @api_key
       }
     end
   end