neo4j-ruby-driver 1.7.5 → 4.4.0.alpha.2

data/ruby/neo4j/driver/internal/async/network_session.rb

@@ -0,0 +1,152 @@
+module Neo4j::Driver
+  module Internal
+    module Async
+      class NetworkSession
+        attr_reader :retry_logic
+
+        def initialize(connection_provider, retry_logic, database_name, mode, bookmark_holder, impersonated_user, fetch_size, logger)
+          @connection_provider = connection_provider
+          @mode = mode
+          @retry_logic = retry_logic
+          @log = Logging::PrefixedLogger.new("[#{hash}]", logger)
+          @bookmark_holder = bookmark_holder
+          @database_name = database_name.database_name
+          @connection_context = NetworkSessionConnectionContext.new(@database_name, @bookmark_holder.bookmark, impersonated_user)
+          @fetch_size = fetch_size
+          @open = Concurrent::AtomicBoolean.new(true)
+        end
+
+        def run_async(query, **config)
+          new_result_cursor = build_result_cursor_factory(query, config).async_result
+          @result_cursor = new_result_cursor
+          new_result_cursor.map_successful_run_completion_async
+        end
+
+        def run_rx(query, **config)
+          new_result_cursor_stage = build_result_cursor_factory(query, config).then_flat(Cursor::ResultCursorFactory.rx_result)
+          @result_cursor_stage = new_result_cursor_stage.rescue {}
+          new_result_cursor_stage
+        end
+
+        def begin_transaction_async(mode = @mode, **config)
+          ensure_session_is_open
+          ensure_no_open_tx_before_starting_tx
+          acquire_connection(mode).then do |connection|
+            ImpersonationUtil.ensure_impersonation_support(connection, connection.impersonated_user)
+            tx = UnmanagedTransaction.new(connection, @bookmark_holder, @fetch_size)
+            tx.begin_async(@bookmark_holder.bookmark, config)
+          end&.tap { |new_transaction| @transaction = new_transaction }
+        end
+
+        def reset_async
+          existing_transaction_or_null
+            .then_accept { |tx| tx&.mark_terminated }
+            .then_flat { @connection_stage }
+            .then_flat do |connection|
+            # there exists an active connection, send a RESET message over it
+            connection&.reset || Util::Futures.completed_with_null
+          end
+        end
+
+        def last_bookmark
+          @bookmark_holder.bookmark
+        end
+
+        def release_connection_async
+          @connection&.release
+        end
+
+        def connection_async
+          @connection
+        end
+
+        def open?
+          @open.true?
+        end
+
+        def close_async
+          return unless @open.make_false
+          # there exists a cursor with potentially unconsumed error, try to extract and propagate it
+          @result_cursor&.discard_all_failure_async
+        ensure
+          close_transaction_and_release_connection
+        end
+
+        def current_connection_open?
+          @connection&.open? # some connection has actually been acquired and it's still open
+        end
+
+        private
+
+        def build_result_cursor_factory(query, config)
+          ensure_session_is_open
+          ensure_no_open_tx_before_running_query
+          connection = acquire_connection(@mode)
+          ImpersonationUtil.ensure_impersonation_support(connection, connection.impersonated_user)
+          connection.protocol.run_in_auto_commit_transaction(connection, query, @bookmark_holder, config,
+                                                             @fetch_size)
+        end
+
+        def acquire_connection(mode)
+          # make sure previous result is fully consumed and connection is released back to the pool
+          @result_cursor&.pull_all_failure_async
+          if @connection&.open?
+            # there somehow is an existing open connection, this should not happen, just a precondition
+            raise Neo4j::Driver::Exceptions::IllegalStateException.new('Existing open connection detected')
+          end
+
+          @connection = @connection_provider.acquire_connection(@connection_context.context_with_mode(mode))
+        end
+
+        def close_transaction_and_release_connection
+          existing_transaction_or_null&.close_async
+        ensure
+          release_connection_async
+        end
+
+        def ensure_no_open_tx_before_running_query
+          ensure_no_open_tx('Queries cannot be run directly on a session with an open transaction; either run from within the transaction or use a different session.')
+        end
+
+        def ensure_no_open_tx_before_starting_tx
+          ensure_no_open_tx('You cannot begin a transaction on a session with an open transaction; either run from within the transaction or use a different session.')
+        end
+
+        def ensure_no_open_tx(error_message)
+          existing_transaction_or_null&.then do
+            raise Neo4j::Driver::Exceptions::TransactionNestingException.new(error_message)
+          end
+        end
+
+        def existing_transaction_or_null
+          @transaction if @transaction&.open?
+        end
+
+        def ensure_session_is_open
+          unless @open.true?
+            raise Neo4j::Driver::Exceptions::ClientException.new('No more interaction with this session are allowed as the current session is already closed.')
+          end
+        end
+
+        # The {@link NetworkSessionConnectionContext#mode} can be mutable for a session connection context
+        class NetworkSessionConnectionContext
+          # This bookmark is only used for rediscovery.
+          # It has to be the initial bookmark given at the creation of the session.
+          # As only that bookmark could carry extra system bookmarks
+          attr_reader :database_name, :mode, :rediscovery_bookmark, :impersonated_user
+
+          def initialize(database_name, bookmark, impersonated_user)
+            @database_name = database_name
+            @rediscovery_bookmark = bookmark
+            @impersonated_user = impersonated_user
+          end
+
+          def context_with_mode(mode)
+            @mode = mode
+            self
+          end
+        end
+      end
+    end
+  end
+end

data/ruby/neo4j/driver/internal/async/outbound/chunk_aware_byte_buf_output.rb

@@ -0,0 +1,110 @@
+module Neo4j::Driver
+  module Internal
+    module Async
+      module Outbound
+        class ChunkAwareByteBufOutput
+          include Packstream::PackStream::Packer
+          include Messaging::Common::CommonValuePacker
+
+          class ChunkBuffer < ::Async::IO::Buffer
+            include Packstream::PackOutput
+            alias write <<
+          end
+
+          def initialize(output, max_chunk_size: Connection::BoltProtocolUtil::DEFAULT_MAX_OUTBOUND_CHUNK_SIZE_BYTES)
+            @output = output
+            @max_chunk_size = verify_max_chunk_size(max_chunk_size)
+            @chunk = ChunkBuffer.new
+          end
+
+          def start
+            assert_not_started
+            @chunk.clear
+          end
+
+          def write_byte(value)
+            ensure_can_fit_in_current_chunk(1)
+            @chunk.write_byte(value)
+            self
+          end
+
+          def write(data)
+            offset = 0
+            length = data.bytesize
+
+            while offset < length
+              # Ensure there is an open chunk, and that it has at least one byte of space left
+              ensure_can_fit_in_current_chunk(1)
+
+              # Write as much as we can into the current chunk
+              amount_to_write = [available_bytes_in_current_chunk, length - offset].min
+
+              @chunk.write(data.byteslice(offset, amount_to_write))
+              offset += amount_to_write
+            end
+
+            self
+          end
+
+          def write_short(value)
+            ensure_can_fit_in_current_chunk(2)
+            @chunk.write_short(value)
+            self
+          end
+
+          def write_int(value)
+            ensure_can_fit_in_current_chunk(4)
+            @chunk.write_int(value)
+            self
+          end
+
+          def write_long(value)
+            ensure_can_fit_in_current_chunk(8)
+            @chunk.write_long(value)
+            self
+          end
+
+          def write_double(value)
+            ensure_can_fit_in_current_chunk(8)
+            @chunk.write_double(value)
+            self
+          end
+
+          def write_message_boundary
+            @output.write_short(0)
+          end
+
+          def write_chunk
+            @output.write_short(@chunk.bytesize)
+            @output.write(@chunk)
+            @chunk.clear
+          end
+
+          alias stop write_chunk
+
+          private
+
+          def ensure_can_fit_in_current_chunk(number_of_bytes)
+            write_chunk if @chunk.bytesize + number_of_bytes > @max_chunk_size
+          end
+
+          def available_bytes_in_current_chunk
+            @max_chunk_size - @chunk.bytesize
+          end
+
+          def assert_not_started
+            raise Neo4j::Driver::Exceptions::IllegalStateException.new('Already started') unless @chunk.empty?
+          end
+
+          def verify_max_chunk_size(max_chunk_size)
+            if max_chunk_size <= 0
+              raise ArgumentError.new("Max chunk size should be > 0, given: #{max_chunk_size}")
+            end
+
+            max_chunk_size
+          end
+        end
+      end
+    end
+  end
+end

data/ruby/neo4j/driver/internal/async/outbound/outbound_message_handler.rb

@@ -0,0 +1,39 @@
+module Neo4j::Driver
+  module Internal
+    module Async
+      module Outbound
+        class OutboundMessageHandler
+          NAME = self.class.name
+
+          def initialize(output, message_format, logger)
+            @output = output
+            @writer = message_format.new_writer(output)
+            @log = logger
+          end
+
+          def handler_added(ctx)
+            @log = Logging::ChannelActivityLogger.new(ctx.channel, @log, self.class)
+          end
+
+          def handler_removed(ctx)
+            @log = nil
+          end
+
+          def encode(msg)
+            @log.debug("C: #{msg}")
+
+            @output.start
+            begin
+              @writer.write(msg)
+            ensure
+              @output.stop
+            end
+
+            @output.write_message_boundary
+            # @log.debug( "C: #{}") if @log.debug_enabled?
+          end
+        end
+      end
+    end
+  end
+end

data/ruby/neo4j/driver/internal/async/pool/channel.rb

@@ -0,0 +1,63 @@
+module Neo4j::Driver
+  module Internal
+    module Async
+      module Pool
+        class Channel < ::Async::Pool::Resource
+          attr :stream
+          attr_accessor :version, :protocol, :message_format, :message_dispatcher
+          attr :attributes # should be attr
+
+          def initialize(address, connector, logger)
+            super()
+            @attributes = Connection::ChannelAttributes.new
+            @stream = Connection::Stream.new(connector.connect(address))
+            @stream.write(Connection::BoltProtocolUtil.handshake_buf)
+            @stream.flush
+            Connection::HandshakeHandler.new(logger).decode(self)
+            stream_reader = Connection::StreamReader.new(@stream)
+            stream_writer = Outbound::ChunkAwareByteBufOutput.new(@stream)
+            @message_dispatcher = Inbound::InboundMessageDispatcher.new(self, logger)
+            @attributes[:message_dispatcher] = @message_dispatcher
+            @outbound_handler = Outbound::OutboundMessageHandler.new(stream_writer, message_format, logger)
+            @common_message_reader = Messaging::Common::CommonMessageReader.new(stream_reader)
+            connector.initialize_channel(self, protocol)
+          end
+
+          def close
+            super unless closed? # Should this be conditional?
+            @stream.close
+          end
+
+          def write(message)
+            @outbound_handler.encode(message)
+          end
+
+          def write_and_flush(message)
+            write(message)
+            @stream.flush
+            ensure_response_handling
+          end
+
+          private
+
+          def bracketless(host)
+            host.delete_prefix('[').delete_suffix(']')
+          end
+
+          def ensure_response_handling
+            # probably should be synchronized
+            return if @handling_active
+            @handling_active = true
+            while @message_dispatcher.queued_handlers_count > 0 do
+              @common_message_reader.read(@message_dispatcher)
+            end
+            @handling_active = false
+          rescue
+            @handling_active = false
+            raise
+          end
+        end
+      end
+    end
+  end
+end

data/ruby/neo4j/driver/internal/async/pool/connection_pool_impl.rb

@@ -0,0 +1,149 @@
+module Neo4j::Driver
+  module Internal
+    module Async
+      module Pool
+        class ConnectionPoolImpl
+          def initialize(connector, settings, logger)
+            @connector = connector
+            @settings = settings
+            @log = logger
+            @address_to_pool_lock = Concurrent::ReentrantReadWriteLock.new
+            @address_to_pool = {}
+            @closed = Concurrent::AtomicBoolean.new
+          end
+
+          def acquire(address)
+            @log.debug("Acquiring a connection from pool towards #{address}")
+
+            assert_not_closed
+            pool = get_or_create_pool(address)
+
+            begin
+              channel = pool.acquire
+            rescue => error
+              process_acquisition_error(pool, address, error)
+            end
+            assert_not_closed(address, channel, pool)
+            NetworkConnection.new(channel, pool, @log)
+          end
+
+          def retain_all(addresses_to_retain)
+            @address_to_pool_lock.with_write_lock do
+              @address_to_pool.each do |address, pool|
+                unless addresses_to_retain.include?(address)
+                  active_channels = @netty_channel_tracker.in_use_channel_count(address)
+                  if active_channels.zero?
+                    # address is not present in updated routing table and has no active connections
+                    # it's now safe to terminate corresponding connection pool and forget about it
+                    @address_to_pool.delete(address)
+                    if pool
+                      @log.info("Closing connection pool towards #{address}, it has no active connections and is not in the routing table registry.")
+                      close_pool_in_background(address, pool)
+                    end
+                  end
+                end
+              end
+            end
+          end
+
+          def in_use_connections(address)
+            @netty_channel_tracker.in_use_channel_count(address)
+          end
+
+          def idle_connections(address)
+            @netty_channel_tracker.idle_channel_count(address)
+          end
+
+          def close
+            if @closed.make_true
+              @address_to_pool_lock.with_write_lock do
+                # We can only shutdown event loop group when all netty pools are fully closed,
+                # otherwise the netty pools might missing threads (from event loop group) to execute clean ups.
+                close_all_pools
+                @address_to_pool.clear
+              end
+            end
+          end
+
+          def open?(address)
+            @address_to_pool_lock.with_read_lock { @address_to_pool.key?(address) }
+          end
+
+          def to_string
+            @address_to_pool_lock.with_read_lock { "ConnectionPoolImpl{ pools=#{@address_to_pool}}" }
+          end
+
+          private
+
+          def process_acquisition_error(pool, server_address, error)
+            if error.is_a?(::Async::TimeoutError)
+              # NettyChannelPool returns future failed with TimeoutException if acquire operation takes more than
+              # configured time, translate this exception to a prettier one and re-throw
+              raise Neo4j::Driver::Exceptions::ClientException.new("Unable to acquire connection from the pool within configured maximum time of #{@settings.connection_acquisition_timeout.inspect}")
+            # elsif pool.closed?
+              # There is a race condition where a thread tries to acquire a connection while the pool is closed by another concurrent thread.
+              # Treat as failed to obtain connection for a direct driver. For a routing driver, this error should be retried.
+              # raise Neo4j::Driver::Exceptions::ServiceUnavailableException, "Connection pool for server #{server_address} is closed while acquiring a connection."
+            else
+              # some unknown error happened during connection acquisition, propagate it
+              raise error
+            end
+          end
+
+          def assert_not_closed(address = nil, channel = nil, pool = nil)
+            if @closed.true?
+              if address
+                pool.release(channel)
+                close_pool_in_background(address, pool)
+                @address_to_pool_lock.with_write_lock { @address_to_pool.delete(address) }
+                assert_not_closed
+              end
+              raise Exceptions::IllegalStateException, Spi::ConnectionPool::CONNECTION_POOL_CLOSED_ERROR_MESSAGE
+            end
+          end
+
+          # for testing only
+          protected def pool(address)
+            @address_to_pool_lock.with_read_lock { @address_to_pool[address] }
+          end
+
+          def new_pool(address)
+            Controller.wrap(limit: @settings.max_connection_pool_size, acquisition_timeout: @settings.connection_acquisition_timeout) { Channel.new(address, @connector, @log) }
+          end
+
+          def get_or_create_pool(address)
+            @address_to_pool_lock.with_read_lock { @address_to_pool[address] } ||
+              @address_to_pool_lock.with_write_lock do
+                new_pool(address)&.tap do |pool|
+                  # before the connection pool is added I can add the metrics for the pool.
+                  # @metrics_listener.put_pool_metrics(pool.object_id, address, self)
+                  @address_to_pool[address] = pool
+                end
+              end
+          end
+
+          def close_pool(pool)
+            pool.close
+          end
+
+          def close_pool_in_background(address, pool)
+            Async do
+              # Close in the background
+              close_pool(pool)
+            rescue => error
+              @log.warn("An error occurred while closing connection pool towards #{address}.", error)
+            end
+          end
+
+          def close_all_pools
+            @address_to_pool.map do |address, pool|
+              @log.info("Closing connection pool towards #{address}")
+              # Wait for all pools to be closed.
+              close_pool(pool)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/ruby/neo4j/driver/internal/async/pool/controller.rb

@@ -0,0 +1,25 @@
+module Neo4j::Driver
+  module Internal
+    module Async
+      module Pool
+        class Controller < ::Async::Pool::Controller
+          def initialize(constructor, limit: nil, concurrency: nil, acquisition_timeout: nil)
+            super(constructor, limit: limit, concurrency: concurrency)
+            @acquisition_timeout = acquisition_timeout
+          end
+
+          def wait_for_resource
+            case @acquisition_timeout
+            when nil
+              super
+            when 0
+              available_resource or raise ::Async::TimeoutError
+            else
+              ::Async::Task.current.with_timeout(@acquisition_timeout) { super }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/ruby/neo4j/driver/internal/async/pool/netty_channel_health_checker.rb

@@ -0,0 +1,87 @@
+module Neo4j::Driver
+  module Internal
+    module Async
+      module Pool
+        class NettyChannelHealthChecker
+          attr_reader :pool_settings, :clock, :logger, :log, :min_creation_timestamp_millis_opt
+
+          def initialize(pool_settings, clock, logger)
+            @pool_settings = pool_settings
+            @clock = clock
+            @log = logger
+            @min_creation_timestamp_millis_opt = java.util.concurrent.atomic.AtomicReference.new(java.util.Optional.empty)
+          end
+
+          def is_healthy(channel)
+            return channel.event_loop.new_succeeded_future(false) if is_too_old?(channel)
+
+            return ping(channel) if has_been_idle_for_too_long?(channel)
+
+            ACTIVE.is_healthy(channel)
+          end
+
+          def on_expired(e, channel)
+            ts = Connection::ChannelAttributes.creation_timestamp(channel)
+
+            # Override current value ONLY if the new one is greater
+            min_creation_timestamp_millis_opt.get_and_update do |prev|
+              java.util.Optional.of(prev.filter(-> (prev_ts) { ts <= prev_ts }.or_else(ts)))
+            end
+          end
+
+          private
+
+          def is_too_old?(channel)
+            creation_timestamp_millis = Connection::ChannelAttributes.creation_timestamp(channel)
+            min_creation_timestamp_millis_opt = min_creation_timestamp_millis_opt.get
+
+            if min_creation_timestamp_millis_opt.present? && creation_timestamp_millis <= min_creation_timestamp_millis_opt.get
+              log.debug("The channel #{channel} is marked for closure as its creation timestamp is older than or equal to the acceptable minimum timestamp: #{creation_timestamp_millis} <= #{min_creation_timestamp_millis_opt.get}")
+              return true
+            end
+
+            if pool_settings.max_connection_lifetime_enabled
+              current_timestamp_millis = clock.millis
+
+              age_millis = current_timestamp_millis - creation_timestamp_millis
+              max_age_millis = pool_settings.max_connection_lifetime
+
+              too_old = age_millis > max_age_millis
+
+              if too_old
+                log.debug("Failed acquire channel #{channel} from the pool because it is too old: #{age_millis} > #{max_age_millis}")
+              end
+              return too_old
+            end
+
+            false
+          end
+
+          def has_been_idle_for_too_long?(channel)
+            if pool_settings.idle_time_before_connection_test_enabled?
+              last_used_timestamp = Connection::ChannelAttributes.last_used_timestamp(channel)
+              if !last_used_timestamp.nil?
+                idle_time = clock.millis - last_used_timestamp
+                idle_too_long = idle_time > pool_settings.idle_time_before_connection_test
+
+                if idle_too_long
+                  log.debug( "Channel #{channel} has been idle for #{idle_time} and needs a ping")
+                end
+
+                return idle_too_long
+              end
+            end
+            false
+          end
+
+          def ping(channel)
+            result = channel.event_loop.new_promise
+            Connection::ChannelAttributes.message_dispatcher.enqueue(Handlers::PingResponseHandler.new(result, channel, logger))
+            channel.write_and_flush(Messaging::Request::ResetMessage::RESET, channel.void_promise)
+            result
+          end
+        end
+      end
+    end
+  end
+end

data/ruby/neo4j/driver/internal/async/pool/netty_channel_pool.rb

@@ -0,0 +1,52 @@
+module Neo4j::Driver
+  module Internal
+    module Async
+      module Pool
+        class NettyChannelPool
+          # Unlimited amount of parties are allowed to request channels from the pool.
+          MAX_PENDING_ACQUIRES = nil
+
+          # Do not check channels when they are returned to the pool.
+          RELEASE_HEALTH_CHECK = false
+
+          attr_reader :id
+
+          def initialize(address, connector, bootstrap, handler, health_check, acquire_timeout_millis, max_connections)
+            java.util.Objects.require_non_null(address)
+            java.util.Objects.require_non_null(connector)
+            java.util.Objects.require_non_null(handler)
+            @id = pool_id(address)
+            @delegate = Java::IoNettyChannelPool::FixedChannelPool.new(bootstrap, handler, health_check,
+                        Java::IoNettyChannelPool::FixedChannelPool::AcquireTimeoutAction::FAIL, acquire_timeout_millis,
+                        max_connections, MAX_PENDING_ACQUIRES, RELEASE_HEALTH_CHECK)
+          end
+
+          def close
+            if closed.compare_and_set(false, true)
+              Util::Futurs.as_completion_stage(delegate.close_async, close_future)
+            end
+            close_future
+          end
+
+          def acquire
+            Util::Futurs.as_completion_stage(delegate.acquire)
+          end
+
+          def release(channel)
+            Util::Futurs.as_completion_stage(delegate.release(channel))
+          end
+
+          def is_closed?
+            closed.get
+          end
+
+          private
+
+          def pool_id(server_address)
+            [server_address.host, server_address.port, self.hash_code]
+          end
+        end
+      end
+    end
+  end
+end