neo4j-core 6.1.6 → 7.0.0.alpha.1

data/lib/neo4j/core/cypher_session/adaptors/bolt.rb

@@ -0,0 +1,331 @@
+require 'neo4j/core/cypher_session/adaptors'
+require 'neo4j/core/cypher_session/adaptors/has_uri'
+require 'neo4j/core/cypher_session/adaptors/bolt/pack_stream'
+require 'neo4j/core/cypher_session/adaptors/bolt/chunk_writer_io'
+require 'neo4j/core/cypher_session/responses/bolt'
+require 'io/wait'
+
+# TODO: Work with `Query` objects
+module Neo4j
+  module Core
+    class CypherSession
+      module Adaptors
+        class Bolt < Base
+          include Adaptors::HasUri
+          default_url('bolt://neo4:neo4j@localhost:7687')
+          validate_uri do |uri|
+            uri.scheme == 'bolt'
+          end
+
+          SUPPORTED_VERSIONS = [1, 0, 0, 0].freeze
+          VERSION = '0.0.1'.freeze
+
+          def initialize(url, options = {})
+            self.url = url
+            @options = options
+
+            open_socket
+          end
+
+          def connect
+            handshake
+
+            init
+
+            message = flush_messages[0]
+            return if message.type == :success
+
+            data = message.args[0]
+            fail "Init did not complete successfully\n\n#{data['code']}\n#{data['message']}"
+          end
+
+          def query_set(transaction, queries, options = {})
+            setup_queries!(queries, transaction, skip_instrumentation: options[:skip_instrumentation])
+
+            if @socket.ready?
+              debug_remaining_buffer
+              fail 'Making query, but expected there to be no buffer remaining!'
+            end
+
+            send_query_jobs(queries)
+
+            build_response(queries, options[:wrap_level] || @options[:wrap_level])
+          end
+
+          def version
+            fail 'should be implemented!'
+          end
+
+          def connected?
+            !!@socket
+          end
+
+          def indexes(session)
+            result = query(session, 'CALL db.indexes()', {}, skip_instrumentation: true)
+
+            result.map do |row|
+              label, property = row.description.match(/INDEX ON :([^\(]+)\(([^\)]+)\)/)[1, 2]
+              {type: row.type.to_sym, label: label.to_sym, properties: [property.to_sym]}
+            end
+          end
+
+          def constraints(session)
+            result = query(session, 'CALL db.indexes()', {}, skip_instrumentation: true)
+
+            result.select { |row| row.type == 'node_unique_property' }.map do |row|
+              label, property = row.description.match(/INDEX ON :([^\(]+)\(([^\)]+)\)/)[1, 2]
+              {type: :uniqueness, label: label.to_sym, properties: [property.to_sym]}
+            end
+          end
+
+          def self.transaction_class
+            require 'neo4j/core/cypher_session/transactions/bolt'
+            Neo4j::Core::CypherSession::Transactions::Bolt
+          end
+
+          instrument(:request, 'neo4j.core.bolt.request', %w(url body)) do |_, start, finish, _id, payload|
+            ms = (finish - start) * 1000
+
+            " #{ANSI::BLUE}BOLT REQUEST:#{ANSI::CLEAR} #{ANSI::YELLOW}#{ms.round}ms#{ANSI::CLEAR} #{payload[:url]}"
+          end
+
+          private
+
+          def build_response(queries, wrap_level)
+            catch(:cypher_bolt_failure) do
+              Responses::Bolt.new(queries, method(:flush_messages), wrap_level: wrap_level).results
+            end.tap do |error_data|
+              handle_failure!(error_data) if !error_data.is_a?(Array)
+            end
+          end
+
+          def handle_failure!(error_data)
+            flush_messages
+
+            send_job do |job|
+              job.add_message(:ack_failure)
+            end
+            fail 'Expected SUCCESS for ACK_FAILURE' if flush_messages[0].type != :success
+
+            fail CypherError.new_from(error_data['code'], error_data['message'])
+          end
+
+          def debug_remaining_buffer
+            logger.debug 'Remaining buffer:'
+
+            i = 0
+            while @socket.ready?
+              i += 1
+              logger.debug "Message set #{i}:"
+              flush_messages
+            end
+          end
+
+          def send_query_jobs(queries)
+            send_job do |job|
+              queries.each do |query|
+                job.add_message(:run, query.cypher, query.parameters || {})
+                job.add_message(:pull_all)
+              end
+            end
+          end
+
+          def new_job
+            Job.new(self)
+          end
+
+          def open_socket
+            @socket = TCPSocket.open(host, port)
+          end
+
+          GOGOBOLT = "\x60\x60\xB0\x17"
+          def handshake
+            log_message :C, :handshake, nil
+
+            sendmsg(GOGOBOLT + SUPPORTED_VERSIONS.pack('l>*'))
+
+            agreed_version = recvmsg(4).unpack('l>*')[0]
+
+            if agreed_version.zero?
+              @socket.shutdown(Socket::SHUT_RDWR)
+              @socket.close
+
+              fail "Couldn't agree on a version (Sent versions #{SUPPORTED_VERSIONS.inspect})"
+            end
+
+            logger.debug "Agreed to version: #{agreed_version}"
+          end
+
+          def init
+            send_job do |job|
+              job.add_message(:init, USER_AGENT_STRING, principal: user, credentials: password, scheme: 'basic')
+            end
+          end
+
+          # Allows you to send messages to the server
+          # Returns an array of Message objects
+          def send_job
+            new_job.tap do |job|
+              yield job
+              log_message :C, :job, job
+              sendmsg(job.chunked_packed_stream)
+            end
+          end
+
+          STREAM_INSPECTOR = lambda do |stream|
+            stream.bytes.map { |byte| byte.to_s(16).rjust(2, '0') }.join(':')
+          end
+
+          def sendmsg(message)
+            log_message :C, message
+
+            @socket.sendmsg(message)
+          end
+
+          def recvmsg(size, timeout = 10)
+            Timeout.timeout(timeout) do
+              @socket.recv(size).tap do |result|
+                log_message :S, result
+              end
+            end
+          rescue Timeout::Error
+            raise "Timed out waiting for #{size} bytes from Neo4j (after #{timeout} seconds)"
+          end
+
+          def flush_messages
+            if structures = flush_response
+              structures.map do |structure|
+                Message.new(structure.signature, *structure.list).tap do |message|
+                  log_message :S, message.type, message.args.join(' ')
+                end
+              end.tap { flush_response }
+            end
+          end
+
+          def log_message(side, *args)
+            if args.size == 1
+              logger.debug "#{side}: #{STREAM_INSPECTOR.call(args[0])}"
+            else
+              type, message = args
+              logger.debug "#{side}: #{ANSI::CYAN}#{type.to_s.upcase}#{ANSI::CLEAR} #{message}"
+            end
+          end
+
+          # Replace with Enumerator?
+          def flush_response
+            if !(header = recvmsg(2)).empty? && (chunk_size = header.unpack('s>*')[0]) > 0
+              log_message :S, :chunk_size, chunk_size
+
+              chunk = recvmsg(chunk_size)
+
+              unpacker = PackStream::Unpacker.new(StringIO.new(chunk))
+
+              [].tap { |r| while arg = unpacker.unpack_value!; r << arg; end }
+            end
+          end
+
+          # Represents messages sent to or received from the server
+          class Message
+            TYPE_CODES = {
+              # client message types
+              init: 0x01,             # 0000 0001 // INIT <user_agent>
+              ack_failure: 0x0E,      # 0000 1110 // ACK_FAILURE
+              reset: 0x0F,            # 0000 1111 // RESET
+              run: 0x10,              # 0001 0000 // RUN <statement> <parameters>
+              discard_all: 0x2F,      # 0010 1111 // DISCARD *
+              pull_all: 0x3F,         # 0011 1111 // PULL *
+
+              # server message types
+              success: 0x70,          # 0111 0000 // SUCCESS <metadata>
+              record: 0x71,           # 0111 0001 // RECORD <value>
+              ignored: 0x7E,          # 0111 1110 // IGNORED <metadata>
+              failure: 0x7F           # 0111 1111 // FAILURE <metadata>
+            }.freeze
+
+            CODE_TYPES = TYPE_CODES.invert
+
+            def initialize(type_or_code, *args)
+              @type_code = Message.type_code_for(type_or_code)
+              fail "Invalid message type: #{@type_code.inspect}" if !@type_code
+              @type = CODE_TYPES[@type_code]
+
+              @args = args
+            end
+
+            def struct
+              PackStream::Structure.new(@type_code, @args)
+            end
+
+            def to_s
+              "#{ANSI::GREEN}#{@type.to_s.upcase}#{ANSI::CLEAR} #{@args.inspect if !@args.empty?}"
+            end
+
+            def packed_stream
+              PackStream::Packer.new(struct).packed_stream
+            end
+
+            def value
+              return if @type != :record
+              @args.map do |arg|
+                # Assuming result is Record
+                field_names = arg[1]
+
+                field_values = arg[2].map do |field_value|
+                  Message.interpret_field_value(field_value)
+                end
+
+                Hash[field_names.zip(field_values)]
+              end
+            end
+
+            attr_reader :type, :args
+
+            def self.type_code_for(type_or_code)
+              type_or_code.is_a?(Integer) ? type_or_code : TYPE_CODES[type_or_code]
+            end
+
+            def self.interpret_field_value(value)
+              if value.is_a?(Array) && (1..3).cover?(value[0])
+                case value[0]
+                when 1
+                  {type: :node, identity: value[1],
+                   labels: value[2], properties: value[3]}
+                end
+              else
+                value
+              end
+            end
+          end
+
+          # Represents a set of messages to send to the server
+          class Job
+            def initialize(session)
+              @messages = []
+              @session = session
+            end
+
+            def add_message(type, *args)
+              @messages << Message.new(type, *args)
+            end
+
+            def chunked_packed_stream
+              io = ChunkWriterIO.new
+
+              @messages.each do |message|
+                io.write(message.packed_stream)
+                io.flush(true)
+              end
+
+              io.rewind
+              io.read
+            end
+
+            def to_s
+              @messages.join(' | ')
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/neo4j/core/cypher_session/adaptors/bolt/chunk_writer_io.rb

@@ -0,0 +1,76 @@
+require 'stringio'
+
+class ChunkWriterIO < StringIO
+  # Writer for chunked data.
+
+  MAX_CHUNK_SIZE = 0xFFFF
+
+  def initialize
+    @output_buffer = []
+    @output_size = 0
+    super
+  end
+
+  # Write some bytes, splitting into chunks if necessary.
+  def write_with_chunking(string)
+    # Kernel.puts "Write!"
+    until string.empty?
+      future_size = @output_size + string.size
+      if future_size >= MAX_CHUNK_SIZE
+        last = MAX_CHUNK_SIZE - @output_size
+        write_buffer!(string[0, last], MAX_CHUNK_SIZE)
+        string = string[last..-1]
+
+        write_without_chunking(buffer_result)
+        clear_buffer!
+      else
+        write_buffer!(string, future_size)
+
+        string = ''
+      end
+    end
+  end
+
+  alias write_without_chunking write
+  alias write write_with_chunking
+
+  def flush(zero_chunk = false)
+    write_without_chunking(buffer_result(zero_chunk))
+    clear_buffer!
+
+    super()
+  end
+
+  # Close the stream.
+  def close(zero_chunk = false)
+    flush(zero_chunk)
+    super
+  end
+
+  # private
+  def write_buffer!(string, size)
+    @output_buffer << string
+    @output_size = size
+  end
+
+  def buffer_result(zero_chunk = false)
+    result = ''
+
+    # Kernel.puts 'result1', result.inspect
+    if !@output_buffer.empty?
+      result << [@output_size].pack('s>*')
+      result.concat(@output_buffer.join)
+    end
+
+    # Kernel.puts 'result2', result.inspect
+    result << "\x00\x00" if zero_chunk
+    # Kernel.puts 'result3', result.inspect
+
+    result
+  end
+
+  def clear_buffer!
+    @output_buffer.clear
+    @output_size = 0
+  end
+end

data/lib/neo4j/core/cypher_session/adaptors/bolt/pack_stream.rb

@@ -0,0 +1,288 @@
+require 'stringio'
+
+module Neo4j
+  module Core
+    # Implements the the PackStream packing and unpacking specifications
+    # as specified by Neo Technology for the Neo4j graph database
+    module PackStream
+      MARKER_TYPES = {
+        C0: nil,
+        C1: [:float, 64],
+        C2: false,
+        C3: true,
+        C8: [:int, 8],
+        C9: [:int, 16],
+        CA: [:int, 32],
+        CB: [:int, 64],
+        CC: [:bytes, 8],
+        CD: [:bytes, 16],
+        CE: [:bytes, 32],
+        D0: [:text, 8],
+        D1: [:text, 16],
+        D2: [:text, 32],
+        D4: [:list, 8],
+        D5: [:list, 16],
+        D6: [:list, 32],
+        D8: [:map, 8],
+        D9: [:map, 16],
+        DA: [:map, 32],
+        DC: [:struct, 8],
+        DD: [:struct, 16],
+        DE: [:struct, 32]
+      }
+      # For efficiency.  Translates directly from bytes to types
+      MARKER_TYPES.keys.each do |key|
+        ord = eval("0x#{key}") # rubocop:disable Lint/Eval
+        MARKER_TYPES[ord] = MARKER_TYPES.delete(key)
+      end
+
+      # Translates directly from types to bytes
+      MARKER_BYTES = MARKER_TYPES.invert
+      MARKER_BYTES.keys.each do |key|
+        MARKER_BYTES.delete(key) if key.is_a?(Array)
+      end
+
+      MARKER_HEADERS = MARKER_TYPES.each_with_object({}) do |(byte, (type, size)), headers|
+        headers[type] ||= {}
+        headers[type][size] = [byte].pack('C')
+      end
+
+      HEADER_PACK_STRINGS = %w(C S L).freeze
+
+      Structure = Struct.new(:signature, :list)
+
+      # Object which holds a Ruby object and can
+      # pack it into a PackStream stream
+      class Packer
+        def initialize(object)
+          @object = object
+        end
+
+        def packed_stream
+          if byte = MARKER_BYTES[@object]
+            pack_array_as_string([byte])
+          else
+            case @object
+            when Date, Time, DateTime then string_stream
+            when Integer, Float, String, Symbol, Array, Structure, Hash
+              send(@object.class.name.split('::').last.downcase + '_stream')
+            end
+          end
+        end
+
+        #   Range Minimum             |  Range Maximum             | Representation | Byte |
+        # ============================|============================|================|======|
+        #  -9 223 372 036 854 775 808 |             -2 147 483 649 | INT_64         | CB   |
+        #              -2 147 483 648 |                    -32 769 | INT_32         | CA   |
+        #                     -32 768 |                       -129 | INT_16         | C9   |
+        #                        -128 |                        -17 | INT_8          | C8   |
+        #                         -16 |                       +127 | TINY_INT       | N/A  |
+        #                        +128 |                    +32 767 | INT_16         | C9   |
+        #                     +32 768 |             +2 147 483 647 | INT_32         | CA   |
+        #              +2 147 483 648 | +9 223 372 036 854 775 807 | INT_64         | CB   |
+
+        INT_HEADERS = MARKER_HEADERS[:int]
+        def integer_stream
+          case @object
+          when -0x10...0x80 # TINY_INT
+            pack_integer_object_as_string
+          when -0x80...-0x10 # INT_8
+            INT_HEADERS[8] + pack_integer_object_as_string
+          when -0x8000...0x8000 # INT_16
+            INT_HEADERS[16] + pack_integer_object_as_string(2)
+          when -0x80000000...0x80000000 # INT_32
+            INT_HEADERS[32] + pack_integer_object_as_string(4)
+          when -0x8000000000000000...0x8000000000000000 # INT_64
+            INT_HEADERS[64] + pack_integer_object_as_string(8)
+          end
+        end
+
+        alias fixnum_stream integer_stream
+        alias bignum_stream integer_stream
+
+        def float_stream
+          MARKER_HEADERS[:float][64] + [@object].pack('G').force_encoding(Encoding::BINARY)
+        end
+
+        #  Marker | Size                                        | Maximum size
+        # ========|=============================================|=====================
+        #  80..8F | contained within low-order nibble of marker | 15 bytes
+        #  D0     | 8-bit big-endian unsigned integer           | 255 bytes
+        #  D1     | 16-bit big-endian unsigned integer          | 65 535 bytes
+        #  D2     | 32-bit big-endian unsigned integer          | 4 294 967 295 bytes
+
+        def string_stream
+          s = @object.to_s
+          s = s.dup if s.frozen?
+          marker_string(0x80, 0xD0, @object.to_s.bytesize) + s.force_encoding(Encoding::BINARY)
+        end
+
+        alias symbol_stream string_stream
+
+        def array_stream
+          marker_string(0x90, 0xD4, @object.size) + @object.map do |e|
+            Packer.new(e).packed_stream
+          end.join
+        end
+
+        def structure_stream
+          fail 'Structure too big' if @object.list.size > 65_535
+          marker_string(0xB0, 0xDC, @object.list.size) + [@object.signature].pack('C') + @object.list.map do |e|
+            Packer.new(e).packed_stream
+          end.join
+        end
+
+        def hash_stream
+          marker_string(0xA0, 0xD8, @object.size) +
+            @object.map do |key, value|
+              Packer.new(key).packed_stream +
+                Packer.new(value).packed_stream
+            end.join
+        end
+
+        def self.pack_arguments(*objects)
+          objects.map { |o| new(o).packed_stream }.join
+        end
+
+        private
+
+        def marker_string(tiny_base, regular_base, size)
+          head_byte = case size
+                      when 0...0x10 then tiny_base + size
+                      when 0x10...0x100 then regular_base
+                      when 0x100...0x10000 then regular_base + 1
+                      when 0x10000...0x100000000 then regular_base + 2
+                      end
+
+          result = [head_byte].pack('C')
+          result += [size].pack(HEADER_PACK_STRINGS[head_byte - regular_base]).reverse if size >= 0x10
+          result
+        end
+
+        def pack_integer_object_as_string(size = 1)
+          bytes = []
+          (0...size).to_a.reverse.inject(@object) do |current, i|
+            bytes << (current / (256**i))
+            current % (256**i)
+          end
+
+          pack_array_as_string(bytes)
+        end
+
+        def pack_array_as_string(a)
+          a.pack('c*')
+        end
+      end
+
+      # Object which holds a stream of PackStream data
+      # and can unpack it
+      class Unpacker
+        def initialize(stream)
+          @stream = stream
+        end
+
+        HEADER_BASE_BYTES = {text: 0xD0, list: 0xD4, struct: 0xDC, map: 0xD8}.freeze
+
+        def unpack_value!
+          return nil if depleted?
+
+          marker = shift_byte!
+
+          if type_and_size = PackStream.marker_type_and_size(marker)
+            type, size = type_and_size
+
+            shift_value_for_type!(type, size, marker)
+          elsif MARKER_TYPES.key?(marker)
+            MARKER_TYPES[marker]
+          else
+            marker >= 0xF0 ? -0x100 + marker : marker
+          end
+        end
+
+        private
+
+        METHOD_MAP = {
+          int: :value_for_int!,
+          float: :value_for_float!,
+          tiny_list: :value_for_list!,
+          list: :value_for_list!,
+          tiny_map: :value_for_map!,
+          map: :value_for_map!,
+          tiny_struct: :value_for_struct!,
+          struct: :value_for_struct!
+        }
+
+        def shift_value_for_type!(type, size, marker)
+          if [:text, :list, :map, :struct].include?(type)
+            offset = marker - HEADER_BASE_BYTES[type]
+            size = shift_stream!(2 << (offset - 1)).reverse.unpack(HEADER_PACK_STRINGS[offset])[0]
+          end
+
+          if [:tiny_text, :text, :bytes].include?(type)
+            shift_stream!(size).force_encoding('UTF-8')
+          else
+            send(METHOD_MAP[type], size)
+          end
+        end
+
+        def value_for_int!(size)
+          r = shift_bytes!(size >> 3).reverse.each_with_index.inject(0) do |sum, (byte, i)|
+            sum + (byte * (256**i))
+          end
+
+          (r >> (size - 1)) == 1 ? (r - (2**size)) : r
+        end
+
+        def value_for_float!(_size)
+          shift_stream!(8).unpack('G')[0]
+        end
+
+        def value_for_map!(size)
+          size.times.each_with_object({}) do |_, r|
+            key = unpack_value!
+            r[key] = unpack_value!
+          end
+        end
+
+        def value_for_list!(size)
+          Array.new(size) { unpack_value! }
+        end
+
+        def value_for_struct!(size)
+          Structure.new(shift_byte!, value_for_list!(size))
+        end
+
+
+        def shift_byte!
+          shift_bytes!(1).first unless depleted?
+        end
+
+        def shift_bytes!(length)
+          result = shift_stream!(length)
+          result && result.bytes.to_a
+        end
+
+        def shift_stream!(length)
+          @stream.read(length) if !depleted? || length.zero?
+        end
+
+        def depleted?
+          @stream.eof?
+        end
+      end
+
+      def self.marker_type_and_size(marker)
+        if (marker_spec = MARKER_TYPES[marker]).is_a?(Array)
+          marker_spec
+        else
+          case marker
+          when 0x80..0x8F then [:tiny_text, marker - 0x80]
+          when 0x90..0x9F then [:tiny_list, marker - 0x90]
+          when 0xA0..0xAF then [:tiny_map, marker - 0xA0]
+          when 0xB0..0xBF then [:tiny_struct, marker - 0xB0]
+          end
+        end
+      end
+    end
+  end
+end