tarantool-client 0.1.0 → 0.3.0

data/lib/tarantool/client/named_requests.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Tarantool
+  class Client
+    # Requests whose space and index resolve through the schema cache. Includers set @link, @schema_cache and
+    # @config, and provide ensure_fork_safe and mark_schema_reloaded.
+    # @api private
+    module NamedRequests
+      private
+
+      # Every schema fetch and attempt draws on one deadline, so the whole call stays within timeout.
+      # Waiting for a connection first means the cache already holds its schema or a newer one.
+      def perform_by_name(space, index, timeout, payload, &)
+        ensure_fork_safe
+        deadline = Transport::Deadline.after(timeout)
+        @link.connection(deadline)
+
+        perform_with_schema_check(space, index, deadline, payload, &)
+      rescue TimeoutError
+        raise TimeoutError, "no response within #{timeout}s"
+      end
+
+      def lookup_space(space, timeout)
+        ensure_fork_safe
+        timeout ||= @config.request_timeout
+        deadline = Transport::Deadline.after(timeout)
+        @link.connection(deadline)
+
+        @schema_cache.space_info(space, deadline)
+      rescue TimeoutError
+        raise TimeoutError, "no response within #{timeout}s"
+      end
+
+      def perform_with_schema_check(space, index, deadline, payload, &)
+        schema, ids = resolve_name(space, index, deadline, payload)
+
+        perform_built(schema, ids, deadline, &)
+      rescue ServerError, UnknownFieldError => e
+        raise if e.is_a?(ServerError) && e.code != Protocol::Constants::ER_WRONG_SCHEMA_VERSION
+
+        mark_schema_reloaded(payload)
+        fresh = @schema_cache.reload(schema, timeout: Transport::Deadline.remaining(deadline))
+        schema, ids = resolve_name(space, index, deadline, payload, fresh)
+        perform_built(schema, ids, deadline, &)
+      end
+
+      def perform_built(schema, ids, deadline)
+        perform_resolved(schema, yield(*ids, schema.space_info(ids.first)), deadline)
+      end
+
+      # The cache returns another schema only when resolving had to reload it.
+      def resolve_name(space, index, deadline, payload, schema = @schema_cache.schema)
+        @schema_cache.resolve(space, index, deadline, schema).tap do |resolved, _ids|
+          mark_schema_reloaded(payload) unless resolved.equal?(schema)
+        end
+      end
+
+      # Only the connection the schema came from: once it is lost, another instance may reuse the
+      # schema version with other ids, so the request fails with it rather than wait for a reconnect.
+      def perform_resolved(schema, request, deadline)
+        timeout = Transport::Deadline.remaining(deadline)
+
+        schema.connection.perform(request, timeout:, schema_version: schema.version)
+      end
+    end
+  end
+end

data/lib/tarantool/client/operations.rb

@@ -3,8 +3,10 @@
 module Tarantool
   class Client
     # Data operations, call, eval and SQL. Includers implement `perform(request, timeout:, on_push: nil)`,
-    # `perform_on_space(space, index, timeout:) { |space_id, index_id| request }`, which returns the response to
-    # the request built by the block, and `lookup_space(space, timeout)`, which returns its SpaceInfo.
+    # `perform_on_space(space, index, timeout:, format: false) { |space_id, index_id, space_info| request }`, which
+    # returns the response to the request built by the block, and `lookup_space(space, timeout)`, which returns its
+    # SpaceInfo. With `format: true` the block gets the SpaceInfo of the schema the request is resolved and
+    # version-checked against, even for numeric ids; otherwise the third argument may be absent.
     # @api private
     module Operations
       def ping(timeout: nil)
@@ -17,43 +19,61 @@ module Tarantool
         lookup_space(space, timeout)
       end
 
+      # `after:` takes a tuple or a position; `fetch_position: true` returns a Page carrying the position instead
+      # of a plain Array.
       def select(space, key = [], index: 0, limit: Protocol::Constants::MAX_LIMIT, offset: 0, iterator: :eq,
-                 after: nil, timeout: nil)
-        perform_on_space(space, index, timeout:) do |space_id, index_id|
-          Protocol::Requests::Select.new(space_id:, index_id:, key: key_tuple(key), limit:, offset:, iterator:, after:)
-        end.data
+                 after: nil, fetch_position: false, as: :tuple, timeout: nil)
+        on_space(space, index, as:, timeout:, page: fetch_position) do |space_id, index_id|
+          Protocol::Requests::Select.new(space_id:, index_id:, key: key_tuple(key), limit:, offset:, iterator:, after:,
+                                         fetch_position:)
+        end
       end
 
-      def insert(space, tuple, timeout: nil)
-        perform_on_space(space, 0, timeout:) do |space_id|
-          Protocol::Requests::Insert.new(space_id:, tuple:)
-        end.data.first
+      def insert(space, tuple, as: :tuple, timeout: nil)
+        on_space(space, 0, as:, timeout:, format: tuple.is_a?(Hash)) do |space_id, _index_id, space_info|
+          Protocol::Requests::Insert.new(space_id:, tuple: tuple_for(tuple, space_info))
+        end.first
       end
 
-      def replace(space, tuple, timeout: nil)
-        perform_on_space(space, 0, timeout:) do |space_id|
-          Protocol::Requests::Replace.new(space_id:, tuple:)
-        end.data.first
+      def replace(space, tuple, as: :tuple, timeout: nil)
+        on_space(space, 0, as:, timeout:, format: tuple.is_a?(Hash)) do |space_id, _index_id, space_info|
+          Protocol::Requests::Replace.new(space_id:, tuple: tuple_for(tuple, space_info))
+        end.first
       end
 
-      def update(space, key, operations, index: 0, timeout: nil)
-        perform_on_space(space, index, timeout:) do |space_id, index_id|
+      def update(space, key, operations, index: 0, as: :tuple, timeout: nil)
+        on_space(space, index, as:, timeout:) do |space_id, index_id|
           Protocol::Requests::Update.new(space_id:, index_id:, key: key_tuple(key), operations:)
-        end.data.first
+        end.first
       end
 
       def upsert(space, tuple, operations, timeout: nil)
-        perform_on_space(space, 0, timeout:) do |space_id|
-          Protocol::Requests::Upsert.new(space_id:, tuple:, operations:)
+        perform_on_space(space, 0, timeout:, format: tuple.is_a?(Hash)) do |space_id, _index_id, space_info|
+          Protocol::Requests::Upsert.new(space_id:, tuple: tuple_for(tuple, space_info), operations:)
         end
 
         nil
       end
 
-      def delete(space, key, index: 0, timeout: nil)
-        perform_on_space(space, index, timeout:) do |space_id, index_id|
+      def delete(space, key, index: 0, as: :tuple, timeout: nil)
+        on_space(space, index, as:, timeout:) do |space_id, index_id|
           Protocol::Requests::Delete.new(space_id:, index_id:, key: key_tuple(key))
-        end.data.first
+        end.first
+      end
+
+      # Deletes every tuple `select(space, key, index:, iterator:)` finds, which IPROTO DELETE can't do on a
+      # non-unique index: selects batch_size tuples at a time, continuing past the last one with `after:`, and
+      # deletes each by primary key. Returns the number of tuples deleted. TREE indexes only. Multikey and
+      # functional indexes are not supported either: tuple-based `after` raises ServerError there, before
+      # deleting anything.
+      #
+      # Outside a transaction it is not atomic: on failure the tuples deleted so far stay deleted, and the
+      # raised Tarantool::Error carries the number confirmed deleted in #deleted (a lower bound). Tuples
+      # inserted meanwhile behind the cursor are not deleted. Inside `transaction { |tx| tx.delete_by(...) }`
+      # it is atomic (memtx needs memtx_use_mvcc_engine). `timeout:` bounds the whole operation; nil gives
+      # each request request_timeout.
+      def delete_by(space, key, index:, iterator: :eq, batch_size: 1000, timeout: nil)
+        BatchDelete.new(self, space, key, index:, iterator:, batch_size:, timeout:).call
       end
 
       # on_push runs in the calling thread for each box.session.push message; an exception
@@ -81,6 +101,22 @@ module Tarantool
 
       private
 
+      # A retry after a schema change calls the block again, so the SpaceInfo kept is the one the response came from.
+      def on_space(space, index, as:, timeout:, format: false, page: false)
+        TupleShape.check(as)
+        space_info = nil
+        needs_format = format || as != :tuple
+        response = perform_on_space(space, index, timeout:, format: needs_format) do |space_id, index_id, info|
+          space_info = info
+          yield space_id, index_id, info
+        end
+        tuples = response.data.map { |tuple| TupleShape.apply(as, tuple, space_info) }
+
+        page ? Page.new(tuples:, position: response.position) : tuples
+      end
+
+      def tuple_for(tuple, space_info) = tuple.is_a?(Hash) ? space_info.to_tuple(tuple) : tuple
+
       def key_tuple(key)
         key.is_a?(Array) ? key : [key]
       end

data/lib/tarantool/client/page.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Tarantool
+  class Client
+    # One page of a select asked with `fetch_position: true`: the tuples (converted by `as:`) and the position to
+    # continue after with `after:`, nil when the page is empty. Opaque: tied to the index and iterator that
+    # produced it.
+    Page = Data.define(:tuples, :position) do
+      include Enumerable
+
+      def each(&) = tuples.each(&)
+
+      def empty? = tuples.empty?
+
+      def size = tuples.size
+    end
+  end
+end

data/lib/tarantool/client/pool/member.rb

@@ -8,11 +8,13 @@ module Tarantool
       # @api private
       class Member
         include Pool::MemberForkSafety
+        include Pool::MemberRole
 
         def initialize(uri, options)
           @uri = uri
           @options = options
           @config = Configuration.build(uri, **options)
+          @instrumentation = Instrumentation.build(@config)
           @lock = Mutex.new
           @changed = ConditionVariable.new
           @closed = false
@@ -75,13 +77,6 @@ module Tarantool
           client.close if closed
         end
 
-        def update(status)
-          @lock.synchronize do
-            @read_only = status['is_ro']
-            @changed.broadcast
-          end
-        end
-
         def wait_for_status
           deadline = Transport::Deadline.after(@config.connect_timeout)
 
@@ -115,7 +110,7 @@ module Tarantool
             attempt += 1
           end
         rescue StandardError => e
-          warn "tarantool pool: gave up connecting to #{address}: #{e.class}: #{e.message}"
+          Diagnostics.warn(@config.logger, "tarantool pool: gave up connecting to #{address}: #{e.class}: #{e.message}")
         end
 
         def try_connect
@@ -125,7 +120,7 @@ module Tarantool
         rescue Error
           false
         rescue StandardError => e
-          warn "tarantool pool: connecting to #{address} failed: #{e.class}: #{e.message}"
+          Diagnostics.warn(@config.logger, "tarantool pool: connecting to #{address} failed: #{e.class}: #{e.message}")
           false
         end
 

data/lib/tarantool/client/pool/member_fork_safety.rb

@@ -9,7 +9,16 @@ module Tarantool
         # to the parent: closing the client in the child only drops its socket copy, dead threads join at once.
         # Nothing inherited may stop the reset, or the child's pool would never start its members.
         def reset_for_fork
-          old_retrier, old_client = @lock.synchronize do
+          old_retrier, old_client = clear_fork_state
+          ThreadJoin.join(old_retrier)
+          close_inherited(old_client)
+          @instrumentation&.notify('fork.tarantool')
+        end
+
+        private
+
+        def clear_fork_state
+          @lock.synchronize do
             retrier = @retrier
             client = @client
             @retrier = nil
@@ -17,12 +26,8 @@ module Tarantool
             @read_only = nil
             [retrier, client]
           end
-          ThreadJoin.join(old_retrier)
-          close_inherited(old_client)
         end
 
-        private
-
         def close_inherited(client)
           client&.close
         rescue StandardError

data/lib/tarantool/client/pool/member_role.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Tarantool
+  class Client
+    class Pool
+      # The role a member's instance reports through box.status. Includers set @lock, @changed, @read_only and
+      # @instrumentation.
+      # @api private
+      module MemberRole
+        private
+
+        def update(status)
+          read_only = status['is_ro']
+          changed = @lock.synchronize do
+            @read_only = read_only
+            @changed.broadcast
+            role_changed?(read_only)
+          end
+
+          @instrumentation&.notify('role_change.tarantool', read_only:) if changed
+        end
+
+        # Compared against the last known role rather than @read_only, which disconnect and fork reset:
+        # a reconnect that brings back the same role is not a change. A status without is_ro is routing
+        # information only, never a role to compare against.
+        def role_changed?(read_only)
+          return false if read_only.nil?
+
+          previous = @known_read_only
+          @known_read_only = read_only
+          !previous.nil? && previous != read_only
+        end
+      end
+    end
+  end
+end

data/lib/tarantool/client/protocol/constants.rb

@@ -17,16 +17,19 @@ module Tarantool
         LIMIT = 0x12
         OFFSET = 0x13
         ITERATOR = 0x14
+        FETCH_POSITION = 0x1f
         KEY = 0x20
         TUPLE = 0x21
         FUNCTION_NAME = 0x22
         USER_NAME = 0x23
         EXPR = 0x27
         OPS = 0x28
+        AFTER_POSITION = 0x2e
         AFTER_TUPLE = 0x2f
         DATA = 0x30
         ERROR_24 = 0x31
         METADATA = 0x32
+        POSITION = 0x35
         SQL_TEXT = 0x40
         SQL_BIND = 0x41
         SQL_INFO = 0x42

data/lib/tarantool/client/protocol/requests/select.rb

@@ -4,11 +4,17 @@ module Tarantool
   class Client
     module Protocol
       module Requests
-        # IPROTO_SELECT: index lookup or scan. `after` continues past a tuple (keyset pagination, 2.11+).
+        # IPROTO_SELECT: index lookup or scan. `after` continues past a tuple (Array) or a position (String, from
+        # a response asked with `fetch_position`); keyset pagination, 2.11+.
         # @api private
-        Select = Data.define(:space_id, :index_id, :key, :limit, :offset, :iterator, :after) do
-          def initialize(iterator:, after: nil, **)
+        Select = Data.define(:space_id, :index_id, :key, :limit, :offset, :iterator, :after, :fetch_position) do
+          def self.cursor?(after) = after.nil? || after.is_a?(Array) || after.is_a?(String)
+
+          def initialize(iterator:, after: nil, fetch_position: false, **)
             raise ArgumentError, "unknown iterator: #{iterator.inspect}" unless Constants::ITERATORS.key?(iterator)
+            unless Select.cursor?(after)
+              raise ArgumentError, "after: expected a tuple (Array) or a position (String), got #{after.inspect}"
+            end
 
             super
           end
@@ -22,8 +28,19 @@ module Tarantool
               Constants::LIMIT => limit,
               Constants::OFFSET => offset,
               Constants::ITERATOR => Constants::ITERATORS.fetch(iterator),
-              Constants::KEY => key
-            }.tap { |body| body[Constants::AFTER_TUPLE] = after unless after.nil? }
+              Constants::KEY => key,
+              **pagination
+            }
+          end
+
+          private
+
+          def pagination
+            keys = {}
+            keys[after.is_a?(String) ? Constants::AFTER_POSITION : Constants::AFTER_TUPLE] = after unless after.nil?
+            keys[Constants::FETCH_POSITION] = true if fetch_position
+
+            keys
           end
         end
       end

data/lib/tarantool/client/protocol/response.rb

@@ -17,6 +17,7 @@ module Tarantool
         def sync = header[Constants::SYNC]
         def schema_version = header[Constants::SCHEMA_VERSION]
         def data = body[Constants::DATA]
+        def position = body[Constants::POSITION]
         def error? = type.anybits?(Constants::TYPE_ERROR)
         def push? = type == Constants::CHUNK
         def event? = type == Constants::EVENT

data/lib/tarantool/client/row.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Tarantool
+  class Client
+    # A tuple as the server returned it, with its fields by name (SpaceInfo#to_hash). Returned for `as: :row`.
+    Row = Data.define(:tuple, :attributes)
+  end
+end

data/lib/tarantool/client/schema.rb

@@ -39,7 +39,7 @@ module Tarantool
       end
 
       def space_info(space)
-        @space_infos.fetch(space_id(space)) { raise SchemaError, "space '#{space}' not found" }
+        @space_infos.fetch(space_id(space))
       end
 
       private
@@ -52,9 +52,10 @@ module Tarantool
       end
 
       def space_id(space)
-        return space if space.is_a?(Integer)
+        id = space.is_a?(Integer) ? space : @space_ids[space.to_s]
+        return id if @space_infos.key?(id)
 
-        @space_ids.fetch(space.to_s) { raise SchemaError, "space '#{space}' not found" }
+        raise SchemaError, "space '#{space}' not found"
       end
 
       def index_id(space_id, index)

data/lib/tarantool/client/space_info.rb

@@ -4,6 +4,8 @@ module Tarantool
   class Client
     # Format and indexes of a space from the loaded schema. `indexes` maps names to IndexInfo in id order.
     SpaceInfo = Data.define(:id, :name, :fields, :indexes) do
+      include TupleMapping
+
       def self.from_tuples(space, index_tuples)
         id, _owner, name, _engine, _field_count, _flags, format = space
         fields = (format || []).map { |raw| SpaceInfo::Field.from_raw(raw) }

data/lib/tarantool/client/stream.rb

@@ -6,15 +6,17 @@ module Tarantool
     # stream's requests in order. Created by Client#transaction.
     class Stream
       include Operations
+      include Instrumented
 
       LOST = 'connection lost during the transaction; the server rolled it back'
       FINISHED = 'transaction is finished'
 
-      def initialize(connection:, schema_cache:, id:, request_timeout:)
+      def initialize(connection:, schema_cache:, id:, request_timeout:, instrumentation: nil)
         @connection = connection
         @schema_cache = schema_cache
         @id = id
         @request_timeout = request_timeout
+        @instrumentation = instrumentation
         @state = :idle
         @pid = Process.pid
       end
@@ -51,34 +53,42 @@ module Tarantool
         abandon if @state == :active
       end
 
-      def perform(request, timeout: nil, schema_version: nil, on_push: nil)
+      def transmit(request, timeout: nil, schema_version: nil, on_push: nil)
         check_active
 
         send_request(request, timeout: timeout || @request_timeout, schema_version:, on_push:)
       end
 
-      def perform_on_space(space, index, timeout: nil, &)
-        return perform(yield(space, index), timeout:) if space.is_a?(Integer) && index.is_a?(Integer)
+      def transmit_on_space(space, index, timeout: nil, format: false, payload: nil, &)
+        return transmit(yield(space, index), timeout:) if !format && space.is_a?(Integer) && index.is_a?(Integer)
 
-        perform_by_name(space, index, timeout || @request_timeout, &)
+        perform_by_name(space, index, timeout || @request_timeout, payload, &)
       end
 
       # Checks for a lost connection first: resolving would otherwise meet the next connection's schema
       # and report a schema change instead.
-      def perform_by_name(space, index, timeout)
+      def perform_by_name(space, index, timeout, payload, &)
         check_connection
         deadline = Transport::Deadline.after(timeout)
-        request = yield(*@schema.resolve(space, index, deadline))
+        request = build_request(space, index, deadline, payload, &)
 
-        perform(request, timeout: remaining(deadline), schema_version: @schema.version)
+        transmit(request, timeout: remaining(deadline), schema_version: @schema.version)
       rescue ServerError => e
         raise unless e.code == Protocol::Constants::ER_WRONG_SCHEMA_VERSION
 
-        @schema.raise_changed(deadline)
+        mark_schema_reloaded(payload) if @schema.refresh?(deadline)
+        raise TransactionError, TransactionSchema::CHANGED
       rescue TimeoutError
         raise TimeoutError, "no response within #{timeout}s"
       end
 
+      def build_request(space, index, deadline, payload)
+        (space_id, index_id), reloaded = @schema.resolve(space, index, deadline)
+        mark_schema_reloaded(payload) if reloaded
+
+        yield(space_id, index_id, @schema.space_info(space_id, deadline))
+      end
+
       def lookup_space(space, timeout)
         check_active
         check_connection
@@ -98,7 +108,7 @@ module Tarantool
       def finish(request)
         @state = :finished
 
-        send_request(request, timeout: @request_timeout)
+        instrumented(request) { send_request(request, timeout: @request_timeout) }
       end
 
       # A stream is bound to its connection's session, so a reconnect can't carry the transaction over.

data/lib/tarantool/client/transaction_schema.rb

@@ -17,21 +17,26 @@ module Tarantool
         @version = version
       end
 
+      # The ids, and whether the cache had to catch up with the BEGIN version first.
       def resolve(space, index, deadline)
-        pinned(deadline).resolve(space, index)
+        cached = @cache.schema
+        schema = pinned(cached, deadline)
+
+        [schema.resolve(space, index), !schema.equal?(cached)]
       end
 
       def space_info(space, deadline)
-        pinned(deadline).space_info(space)
+        pinned(@cache.schema, deadline).space_info(space)
       end
 
       # A request rejected for its schema version did not run, so the transaction still rolls back
-      # cleanly. The reload spares the retried transaction the same rejection.
-      def raise_changed(deadline)
+      # cleanly. The reload spares the retried transaction the same rejection. Returns whether it reloaded.
+      def refresh?(deadline)
         schema = @cache.schema
-        reload(schema, deadline) if schema.version == version
+        return false unless schema.version == version
 
-        raise TransactionError, CHANGED
+        reload(schema, deadline)
+        true
       end
 
       private
@@ -39,8 +44,7 @@ module Tarantool
       # A cache older than BEGIN was stale before the transaction started and may catch up;
       # anything newer means the schema changed after BEGIN. A schema of another connection means
       # this one was lost, and the versions of two instances can't be compared.
-      def pinned(deadline)
-        schema = @cache.schema
+      def pinned(schema, deadline)
         raise TransactionError, Stream::LOST unless schema.connection.equal?(@connection)
 
         schema = reload(schema, deadline) if schema.version < version

data/lib/tarantool/client/transport/connection.rb

@@ -58,6 +58,16 @@ module Tarantool
           !@pending.failure.nil?
         end
 
+        def in_flight = @pending.size
+
+        # True once every request got its answer, false at the deadline.
+        def wait_idle(timeout) = @pending.wait_empty(timeout)
+
+        # Ends the connection on the client's initiative with a reason the requests and the disconnect handler see.
+        def shut_down(reason) = break_connection(reason)
+
+        def failure = @pending.failure
+
         private
 
         def establish