xitdb 0.1.0

data/lib/xitdb/core/bytes.rb

@@ -0,0 +1,181 @@
+# frozen_string_literal: true
+
+module XitDB
+  module Core
+    # Reads and writes inline or external byte payloads.
+    module Bytes
+      CHUNK_SIZE = 64 * 1024
+
+      # A bounded reader for one immutable byte payload.
+      class Source
+        attr_reader :size, :format_tag
+
+        def initialize(database, location, size, format_tag, bound)
+          @database = database
+          @location = location
+          @size = size
+          @format_tag = format_tag
+          @bound = bound
+          freeze
+        end
+
+        def read(position, length)
+          @database.check_open!
+          unless position.is_a?(Integer) && length.is_a?(Integer)
+            raise TypeError, "byte positions and lengths must be Integers"
+          end
+          if position.negative? || length.negative? || position + length > size
+            raise InvalidValueError, "truncated byte payload"
+          end
+
+          location = @location
+          if location.is_a?(String)
+            Format.slice(location, position, length)
+          else
+            @database.read_at(location + position, length, bound: @bound)
+          end
+        end
+      end
+      private_constant :Source
+
+      def self.open(database, slot, bound:)
+        location, size, format_tag = parse(database, slot, bound)
+        Source.new(database, location, size, format_tag, bound)
+      end
+
+      def self.append(transaction, size:, format_tag: nil, prefix: "".b, representation: :automatic, &reader)
+        transaction.check_active!
+        validate_source(size, prefix, representation, format_tag)
+        format_tag = format_tag&.b
+        total_size = prefix.bytesize + size
+        raise RangeError, "byte payload exceeds unsigned 64-bit size" if total_size >= (1 << 64)
+
+        capacity = format_tag ? 6 : 8
+        captured = capture_inline(size, prefix, representation, capacity, &reader)
+        if captured && !captured.include?("\0")
+          payload = captured.ljust(capacity, "\0") + (format_tag || "")
+          return Slot.new(tag: :short_bytes, value: Format.unsigned(payload), full: !format_tag.nil?)
+        end
+
+        offset = transaction.append([total_size].pack("Q>"))
+        if captured
+          transaction.append(captured)
+        else
+          append_source(transaction, prefix, size, &reader)
+        end
+        transaction.append(format_tag) if format_tag
+        Slot.new(tag: :bytes, value: offset, full: !format_tag.nil?)
+      end
+
+      def self.validate_source(size, prefix, representation, format_tag)
+        raise TypeError, "size must be an Integer" unless size.is_a?(Integer)
+        raise ArgumentError, "size must be nonnegative" if size.negative?
+        raise TypeError, "prefix must be a String" unless prefix.is_a?(String)
+        raise ArgumentError, "unknown byte representation" unless %i[automatic external].include?(representation)
+        return if format_tag.nil? || (format_tag.is_a?(String) && format_tag.bytesize == 2)
+
+        raise ArgumentError, "format_tag must contain two bytes"
+      end
+
+      def self.capture_inline(size, prefix, representation, capacity, &)
+        return nil unless representation == :automatic && prefix.bytesize + size <= capacity
+
+        prefix.b + (size.zero? ? "".b : read_chunk(0, size, &))
+      end
+
+      def self.append_source(transaction, prefix, size, &)
+        position = 0
+        while position < prefix.bytesize
+          chunk = Format.slice(prefix, position, CHUNK_SIZE).b
+          transaction.append(chunk)
+          position += chunk.bytesize
+        end
+        position = 0
+        while position < size
+          length = [size - position, CHUNK_SIZE].min
+          transaction.append(read_chunk(position, length, &))
+          position += length
+        end
+      end
+      private_class_method :validate_source, :capture_inline, :append_source
+
+      def self.read_chunk(position, length)
+        chunk = yield(position, length) if block_given?
+        raise EOFError, "byte source ended before its declared size" if chunk.nil?
+        raise TypeError, "byte source must return a String" unless chunk.is_a?(String)
+        raise EOFError, "byte source ended before its declared size" if chunk.bytesize < length
+        raise ArgumentError, "byte source returned more than requested" if chunk.bytesize > length
+
+        chunk.b
+      end
+      private_class_method :read_chunk
+
+      def self.parse(database, slot, bound)
+        database.check_open!
+        case slot.tag
+        when :short_bytes
+          payload = [slot.value].pack("Q>")
+          capacity = slot.full ? 6 : 8
+          data = (Format.slice(payload, 0, capacity).split("\0", 2).first || "".b).freeze
+          tag = Format.slice(payload, 6, 2).freeze if slot.full
+          [data, data.bytesize, tag]
+        when :bytes
+          size = Format.unsigned(database.read_at(slot.value, 8, bound: bound))
+          offset = slot.value + 8
+          tail = database.read_at(offset + size, slot.full ? 2 : 0, bound: bound)
+          tag = tail.freeze if slot.full
+          [offset, size, tag]
+        else
+          raise InvalidValueError, "slot does not contain bytes"
+        end
+      end
+      private_class_method :parse
+
+      # Return inline data or its file offset, without materializing a scalar.
+      def self.locate(database, slot, bound:)
+        unless %i[bytes short_bytes].include?(slot.tag) && !slot.full
+          raise InvalidValueError, "stream requires untagged bytes"
+        end
+
+        location, size = parse(database, slot, bound)
+        [location.is_a?(String) ? location.dup : location, size]
+      end
+
+      def self.copy(source, size:, database:, transaction:)
+        if size <= 8
+          bytes = source.read(size)
+          raise EOFError, "byte source ended before its declared size" unless bytes
+
+          return database.bytes(bytes)
+        end
+
+        append(transaction, size: size, representation: :external) do |_position, length|
+          chunk = "".b
+          while chunk.bytesize < length
+            part = source.read(length - chunk.bytesize)
+            raise EOFError, "byte source ended before its declared size" if part.nil? || part.empty?
+
+            chunk << part
+          end
+          chunk
+        end
+      end
+    end
+
+    # An immutable byte payload and its optional format tag.
+    class ByteValue
+      attr_reader :bytes, :format_tag
+
+      def initialize(bytes, format_tag: nil)
+        raise TypeError, "bytes must be a String" unless bytes.is_a?(String)
+        unless format_tag.nil? || (format_tag.is_a?(String) && format_tag.bytesize == 2)
+          raise ArgumentError, "format_tag must contain two bytes"
+        end
+
+        @bytes = bytes.b.freeze
+        @format_tag = format_tag&.b&.freeze
+        freeze
+      end
+    end
+  end
+end

data/lib/xitdb/core/compaction.rb

@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+require_relative "offset_map"
+
+module XitDB
+  module Core
+    # Copies reachable records into a new database.
+    class Compaction
+      INLINE_TAGS = %i[none uint int float short_bytes].freeze
+      BLOCK_SIZE = 16 * Slot::SIZE
+
+      def initialize(source, cursor, transaction, offsets)
+        @source = source
+        @bound = cursor.bound
+        @transaction = transaction
+        @offsets = offsets
+        @stack = []
+      end
+
+      def copy(slot)
+        result = remap(slot)
+        @stack.pop&.call until @stack.empty?
+        result
+      end
+
+      private
+
+      def read(offset, size)
+        @source.read_at(offset, size, bound: @bound)
+      end
+
+      def parse(bytes)
+        slot = Slot.parse(bytes)
+        raise CorruptionError, "unknown compaction slot tag" unless Slot::TAGS.include?(slot.tag)
+
+        slot
+      end
+
+      def remap(slot)
+        return slot if INLINE_TAGS.include?(slot.tag)
+
+        Slot.new(tag: slot.tag, full: slot.full, value: remap_pointer(slot))
+      end
+
+      def remap_pointer(slot)
+        case slot.tag
+        when :bytes then copy_bytes(slot)
+        when :index, :hash_map, :hash_set then reserve_slots(slot.value, BLOCK_SIZE, 0, 16)
+        when :counted_hash_map, :counted_hash_set then reserve_slots(slot.value, BLOCK_SIZE + 8, 8, 16)
+        when :array_list then reserve(slot.value, 16) { |source, target| array_header(source, target) }
+        when :linked_array_list then reserve(slot.value, 16) do |source, target|
+          tree_header(source, target, false)
+        end
+        when :sorted_map, :sorted_set then reserve(slot.value, 16) do |source, target|
+          tree_header(source, target, true)
+        end
+        when :kv_pair then reserve_slots(slot.value, @source.header.hash_size + 18,
+                                         @source.header.hash_size, 2)
+        when :sorted_kv_pair then reserve_slots(slot.value, 18, 0, 2)
+        else raise CorruptionError, "unknown compaction slot tag"
+        end
+      end
+
+      def reserve_slots(source, size, start, count)
+        reserve(source, size) { |origin, target| slots(origin, target, start, count) }
+      end
+
+      def reserve(source, size, cache: true)
+        mapped = @offsets.fetch(source) if cache
+        return mapped if mapped
+
+        bytes = read(source, size)
+        target = @transaction.append(bytes)
+        @offsets.store(source, target) if cache
+        @stack << -> { yield(source, target) }
+        target
+      end
+
+      def copy_bytes(slot)
+        mapped = @offsets.fetch(slot.value)
+        return mapped if mapped
+
+        source = Bytes.open(@source, slot, bound: @bound)
+        copied = Bytes.append(@transaction, size: source.size, format_tag: source.format_tag,
+                                            representation: :external) do |position, length|
+          source.read(position,
+                      length)
+        end
+        target = copied.value
+        @offsets.store(slot.value, target)
+        target
+      end
+
+      def slots(source, target, start, count)
+        count.times do |index|
+          offset = start + (index * Slot::SIZE)
+          slot = parse(read(source + offset, Slot::SIZE))
+          @transaction.write_at(target + offset, remap(slot).to_bytes)
+        end
+      end
+
+      def array_header(source, target)
+        size, pointer = Format.pair(read(source, 16), "Q>Q>")
+        shift = [((size - 1).bit_length - 1) / 4, 0].max
+        mapped = array_index(pointer, size, shift)
+        @transaction.write_at(target, [size, mapped].pack("Q>Q>"))
+      end
+
+      def array_index(source, size, shift)
+        return reserve_slots(source, BLOCK_SIZE, 0, 16) if size == 1 << ((shift + 1) * 4)
+
+        reserve(source, BLOCK_SIZE, cache: false) { |origin, target| partial_array(origin, target, size, shift) }
+      end
+
+      def partial_array(source, target, size, shift)
+        # Partial blocks can be shared by arrays with different visible sizes.
+        # Copy them separately and clear every slot outside this array's size.
+        @transaction.write_at(target, "\0".b * BLOCK_SIZE)
+        child_size = 1 << (shift * 4)
+        16.times do |index|
+          break unless size.positive?
+
+          slot = parse(read(source + (index * Slot::SIZE), Slot::SIZE))
+          mapped = if shift.zero? || slot.empty?
+                     remap(slot)
+                   else
+                     raise CorruptionError, "invalid array branch" unless slot.tag == :index
+
+                     Slot.new(tag: :index, full: slot.full,
+                              value: array_index(slot.value, [size, child_size].min, shift - 1))
+                   end
+          @transaction.write_at(target + (index * Slot::SIZE), mapped.to_bytes)
+          size -= [size, child_size].min
+        end
+      end
+
+      def tree_header(source, target, sorted)
+        size, pointer = Format.pair(read(source, 16), "Q>Q>")
+        @transaction.write_at(target, [size, tree_node(pointer, sorted)].pack("Q>Q>"))
+      end
+
+      def tree_node(source, sorted)
+        mapped = @offsets.fetch(source)
+        return mapped if mapped
+
+        kind, count = Format.pair(read(source, 2), "CC")
+        raise CorruptionError, "invalid compaction B-tree node" unless kind <= 1 && count <= 16
+
+        size = 2 + BLOCK_SIZE + (if kind == 1
+                                   128 + (sorted ? BLOCK_SIZE : 0)
+                                 else
+                                   0
+                                 end)
+        if kind.zero?
+          reserve_slots(source, size, 2, 16)
+        else
+          reserve(source, size) { |origin, target| tree_branch(origin, target, sorted) }
+        end
+      end
+
+      def tree_branch(source, target, sorted)
+        16.times do |index|
+          offset = 2 + (index * Slot::SIZE)
+          slot = parse(read(source + offset, Slot::SIZE))
+          slot = Slot.new(tag: :index, full: slot.full, value: tree_node(slot.value, sorted)) if slot.tag == :index
+          @transaction.write_at(target + offset, slot.to_bytes)
+        end
+        slots(source, target, 2 + BLOCK_SIZE, 16) if sorted
+      end
+    end
+  end
+end

data/lib/xitdb/core/crc64.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module XitDB
+  module Core
+    # Calculates the checksum used by root trailers.
+    module CRC64
+      MASK = 0xffffffffffffffff
+      POLYNOMIAL = 0xc96c5795d7870f42
+
+      def self.call(bytes)
+        raise TypeError, "expected String, got #{bytes.class}" unless bytes.is_a?(String)
+
+        crc = MASK
+        bytes.each_byte do |byte|
+          crc ^= byte
+          8.times { crc = (crc >> 1) ^ (crc.odd? ? POLYNOMIAL : 0) }
+        end
+        crc ^ MASK
+      end
+    end
+  end
+end

data/lib/xitdb/core/cursor.rb

@@ -0,0 +1,201 @@
+# frozen_string_literal: true
+
+module XitDB
+  module Core
+    # Tracks a bounded collection slot and its ancestor path.
+    class Cursor
+      attr_reader :database, :transaction
+
+      def initialize(database:, slot:, bound:, transaction: nil)
+        @database = database
+        @slot = slot
+        @bound = bound
+        @transaction = transaction
+        @valid = true
+        @children = ObjectSpace::WeakMap.new
+      end
+
+      def slot
+        database.check_open!
+        # @type var path: Array[Cursor]
+        path = []
+        ancestor = self
+        loop do
+          transaction&.check_cursor!(ancestor)
+          raise ExpiredViewError, "cursor ancestor was replaced" unless ancestor.valid
+          break unless ancestor.parent
+
+          path << ancestor
+          ancestor = ancestor.parent
+        end
+        value = ancestor.local_slot
+        limit = transaction ? transaction.length : @bound
+        path.reverse_each do |child|
+          reader = self.class.new(database: database, slot: value, bound: limit)
+          value = reader.collection_get(child.selector)
+          raise ExpiredViewError, "cursor element was removed" unless value
+
+          child.instance_variable_set(:@slot, value)
+        end
+        value
+      end
+
+      def bound
+        slot
+        transaction ? transaction.length : @bound
+      end
+
+      def replace(slot)
+        database.check_open!
+        raise ReadOnlyError, "cursor is read-only" unless transaction
+
+        transaction.check_replacement!(self, slot)
+        self.slot
+        parent = @parent
+        parent&.collection_replace(@selector, slot, owner: self)
+        @slot = slot
+        invalidate_children
+        self
+      end
+
+      def child(kind:, selector:)
+        unless %i[array_list linked_array_list sorted_map sorted_set].include?(kind) || HashMap::TAGS.include?(kind)
+          raise ArgumentError, "unsupported child kind: #{kind.inspect}"
+        end
+        return nil if slot.tag == :none
+
+        if kind == :array_list
+          ArrayList.new(self).cursor_at(integer_selector(selector))
+        elsif kind == :linked_array_list
+          LinkedArrayList.new(self).cursor_at(integer_selector(selector))
+        elsif %i[sorted_map sorted_set].include?(kind)
+          SortedMap.new(self).cursor_at(string_selector(selector))
+        else
+          HashMap.from_cursor(self).cursor_at(string_selector(selector))
+        end
+      end
+
+      # Collection updates keep child paths live while replacing copied ancestors.
+      def update_collection(slot, propagate: true)
+        transaction = self.transaction
+        raise ReadOnlyError, "cursor is read-only" unless transaction
+
+        transaction.check_replacement!(self, slot)
+        self.slot
+        @slot = slot
+        if propagate
+          child = self
+          while child.parent
+            parent = child.parent
+            parent.collection_replace(child.selector, child.local_slot, collection: true, propagate: false)
+            child = parent
+          end
+        end
+        self
+      end
+
+      def array_child(index, slot)
+        cached_child(index, slot)
+      end
+
+      def hash_child(hash_bytes, slot)
+        cached_child(hash_bytes, slot)
+      end
+
+      def cached_child(index, slot)
+        @children.each_pair do |child, selector|
+          next unless selector == index && child.valid
+
+          begin
+            child.slot
+          rescue ExpiredViewError
+            next
+          end
+          return child
+        end
+        child = if transaction
+                  transaction.cursor(slot: slot)
+                else
+                  self.class.new(database: database, slot: slot, bound: bound)
+                end
+        child.instance_variable_set(:@parent, self)
+        child.instance_variable_set(:@selector, index)
+        @children[child] = index
+        child
+      end
+
+      def invalidate_children(index: nil, from: nil, except: nil)
+        @children.each_pair do |child, selector|
+          next if child.equal?(except)
+          next if index && selector != index
+          next if from && selector < from
+
+          child.instance_variable_set(:@valid, false)
+        end
+      end
+
+      # These states are internal transaction snapshots, including invalid views.
+      def checkpoint_state
+        [@slot, @valid]
+      end
+
+      def restore_checkpoint_state(state)
+        @slot, @valid = state
+      end
+
+      def read_bytes
+        database.read_bytes(slot, bound: bound)
+      end
+
+      protected
+
+      def collection_get(selector)
+        target = collection
+        if target.is_a?(ArrayList) || target.is_a?(LinkedArrayList)
+          target.get(integer_selector(selector))
+        else
+          target.get(string_selector(selector))
+        end
+      end
+
+      def collection_replace(selector, slot, owner: nil, collection: false, propagate: true)
+        target = self.collection
+        if target.is_a?(ArrayList) || target.is_a?(LinkedArrayList)
+          target.replace_child(integer_selector(selector), slot, owner: owner, collection: collection,
+                                                                 propagate: propagate)
+        else
+          target.replace_child(string_selector(selector), slot, owner: owner, collection: collection,
+                                                                propagate: propagate)
+        end
+      end
+
+      def integer_selector(selector)
+        raise TypeError, "array index must be an Integer" unless selector.is_a?(Integer)
+
+        selector
+      end
+
+      def string_selector(selector)
+        raise TypeError, "collection key must be a String" unless selector.is_a?(String)
+
+        selector
+      end
+
+      attr_reader :parent, :selector, :valid
+
+      def collection
+        case slot.tag
+        when :array_list then ArrayList.new(self)
+        when :linked_array_list then LinkedArrayList.new(self)
+        when :sorted_map, :sorted_set then SortedMap.new(self)
+        when :hash_map, :hash_set, :counted_hash_map, :counted_hash_set then HashMap.from_cursor(self)
+        else raise InvalidValueError, "slot does not contain a supported collection"
+        end
+      end
+
+      def local_slot
+        @slot
+      end
+    end
+  end
+end