xitdb 0.1.0

data/lib/xitdb/core/database.rb

@@ -0,0 +1,359 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require_relative "compaction"
+require_relative "replication"
+
+module XitDB
+  module Core
+    # Coordinates core storage, transactions, and committed roots.
+    class Database
+      include Replication
+
+      ROOT_OFFSET = 20
+      DATA_OFFSET = 37
+      TRAILER_SIZE = 25
+      MINIMUM_SIZE = 62
+
+      attr_reader :header
+
+      def self.open(path, readonly: false, **)
+        store = FileStore.open(path, readonly: readonly)
+        begin
+          database = new(store: store, **)
+        rescue Exception # rubocop:disable Lint/RescueException -- Cleanup must also preserve nonstandard exceptions.
+          store.send(:close_unreturned)
+          raise
+        end
+        return database unless block_given?
+
+        begin
+          yield database
+        ensure
+          database.close
+        end
+      end
+
+      def self.memory(hash_size: 20, hash_id: "sha1", file_id: nil)
+        new(store: MemoryStore.new, hash_size: hash_size, hash_id: hash_id, file_id: file_id)
+      end
+
+      def self.copy_replica(to:, source_io:, length:)
+        Replication.copy_replica(to: to, source_io: source_io, length: length)
+      end
+
+      def initialize(store:, hash_size: nil, hash_id: nil, file_id: nil, lock_timeout: 5.0, initialize_empty: true,
+                     require_empty: false)
+        @store = store
+        @lock_timeout = lock_timeout
+        @needs_reopen = false
+        @transaction = nil
+        initialize_file = lambda do
+          prepare_file(hash_size, hash_id, file_id, initialize_empty, require_empty)
+        end
+        if store.readonly?
+          initialize_file.call
+        else
+          store.with_exclusive_lock(timeout: lock_timeout, &initialize_file)
+        end
+      rescue EOFError => e
+        raise CorruptionError, e.message
+      end
+
+      def close
+        @store.close
+      end
+
+      def closed?
+        @store.closed?
+      end
+
+      def with_operation(&)
+        @store.with_operation(&)
+      end
+
+      def storage_identity
+        check_open!
+        @store.storage_identity
+      end
+
+      def needs_reopen?
+        @needs_reopen
+      end
+
+      def check_open!
+        raise ClosedError, "database is closed" if closed?
+      end
+
+      def committed_size
+        committed_state.first
+      end
+
+      def compact(to:)
+        with_operation do
+          # @type var offsets_for_cleanup: OffsetMap?
+          offsets_for_cleanup = nil
+          # @type var target_for_cleanup: Database?
+          target_for_cleanup = nil
+          source = root_cursor
+          slot = source.slot
+          history = slot.tag == :array_list
+          slot = ArrayList.new(source).get(-1) || Slot.new if history
+          store = FileStore.open(to)
+          begin
+            raise ArgumentError, "compaction target is the source file" if store.storage_identity == storage_identity
+
+            file_id = random_id
+            file_id = random_id while file_id == header.file_id
+            target = Database.new(store: store, hash_id: header.hash_id, hash_size: header.hash_size, file_id: file_id,
+                                  lock_timeout: @lock_timeout, require_empty: true)
+            target_for_cleanup = target
+            offsets = OffsetMap.new
+            offsets_for_cleanup = offsets
+            unless slot.empty?
+              copy_compacted_root(target, store, file_id, source, slot, history, offsets)
+              published = true
+            end
+            offsets.close
+            offsets_for_cleanup = nil
+            completed = true
+            target
+          ensure
+            cleanup_compaction(store, target_for_cleanup, offsets_for_cleanup, published) unless completed
+          end
+        end
+      end
+
+      def root_cursor
+        size, slot = committed_state
+        Cursor.new(database: self, slot: slot, bound: size)
+      end
+
+      def transaction
+        check_open!
+        raise TransactionStateError, "database requires reopening" if @needs_reopen
+
+        callback_completed = false
+        # @type var callback_error: Exception?
+        callback_error = nil
+        # @type var outcome: :not_committed | :committed | :unknown
+        outcome = :not_committed
+        @store.with_exclusive_lock(timeout: @lock_timeout, on_release_error: ->(_error) { @needs_reopen = true }) do
+          raise TransactionStateError, "database requires reopening" if @needs_reopen
+
+          size, slot = committed_state
+          @store.truncate(size) if @store.length > size
+          transaction = Transaction.new(self, @store, slot)
+          @transaction = transaction
+          begin
+            begin
+              yield transaction.root
+            rescue Exception => e # rubocop:disable Lint/RescueException -- Cleanup must also preserve nonstandard exceptions.
+              callback_error = e
+              raise
+            end
+            callback_completed = true
+            unless transaction.cancelled?
+              publish(transaction.root.slot, random_id)
+              outcome = :committed
+            end
+          ensure
+            finish_transaction(transaction, size, outcome, callback_completed, $!) # rubocop:disable Style/SpecialGlobalVars -- Preserve the pending exception during cleanup.
+          end
+          root_cursor
+        end
+      rescue Exception => e # rubocop:disable Lint/RescueException -- Cleanup must also preserve nonstandard exceptions.
+        handle_transaction_error(e, callback_error, callback_completed, outcome || :not_committed)
+      end
+
+      def handle_transaction_error(error, callback_error, callback_completed, outcome)
+        if callback_error && !error.equal?(callback_error)
+          @needs_reopen = true
+          raise callback_error, cause: callback_error.cause
+        end
+        raise error if error.is_a?(CommitError) || !callback_completed
+
+        @needs_reopen = true
+        raise CommitError.new(error.message, outcome: outcome), cause: error
+      end
+      private :handle_transaction_error
+
+      def bytes(data, format_tag: nil)
+        transaction = active_transaction
+        value = ByteValue.new(data, format_tag: format_tag)
+        Bytes.append(transaction, size: value.bytes.bytesize, format_tag: value.format_tag) do |position, length|
+          Format.slice(value.bytes, position, length)
+        end
+      end
+
+      def read_bytes(slot, bound: nil)
+        check_open!
+        bound ||= committed_size
+        source = Bytes.open(self, slot, bound: bound)
+        ByteValue.new(source.read(0, source.size), format_tag: source.format_tag)
+      end
+
+      def read_at(offset, size, bound:)
+        check_open!
+        check_range!(offset, size, bound)
+        @store.read_at(offset, size)
+      rescue EOFError => e
+        raise CorruptionError, e.message
+      end
+
+      private
+
+      def copy_compacted_root(target, store, file_id, source, slot, history, offsets)
+        target.transaction do |root|
+          current = Header.parse(store.read_at(0, Header::SIZE))
+          unless current.file_id == file_id && root.bound == MINIMUM_SIZE && root.slot.empty?
+            raise ArgumentError, "compaction target changed before copying"
+          end
+
+          transaction = root.transaction
+          raise TransactionStateError, "missing compaction transaction" unless transaction
+
+          copier = Compaction.new(self, source, transaction, offsets)
+          if history
+            ArrayList.new(root).append(copier.copy(slot))
+          else
+            root.replace(copier.copy(slot))
+          end
+        end
+      end
+
+      def cleanup_compaction(store, target, offsets, published)
+        begin
+          offsets&.close
+        rescue Exception # rubocop:disable Lint/RescueException -- Cleanup must also preserve nonstandard exceptions.
+          nil
+        end
+        target&.restore_empty_target unless published
+        store.send(:close_unreturned)
+      end
+
+      def finish_transaction(transaction, size, outcome, callback_completed, pending_error)
+        transaction.expire!
+        @transaction = nil
+        # Cleanup must preserve caller exceptions and nonlocal exits.
+        return unless outcome == :not_committed && !@needs_reopen
+
+        begin
+          @store.truncate(size)
+        rescue Exception => e # rubocop:disable Lint/RescueException -- Cleanup must also preserve nonstandard exceptions.
+          @needs_reopen = true
+          if callback_completed && pending_error.nil?
+            raise CommitError.new(e.message, outcome: :not_committed), cause: e
+          end
+        end
+      end
+
+      def prepare_file(hash_size, hash_id, file_id, initialize_empty, require_empty)
+        raise ArgumentError, "compaction target must be empty" if require_empty && !@store.empty?
+
+        if @store.empty?
+          raise CorruptionError, "empty read-only database" if @store.readonly?
+          raise CorruptionError, "existing empty database" unless initialize_empty
+
+          initialize_empty_file(hash_size, hash_id, file_id, require_empty)
+        else
+          @header = Header.parse(@store.read_at(0, Header::SIZE))
+          check_header(:hash_size, hash_size, @header.hash_size)
+          check_header(:hash_id, hash_id, @header.hash_id)
+          check_header(:file_id, file_id, @header.file_id)
+          committed_state
+        end
+      end
+
+      def initialize_empty_file(hash_size, hash_id, file_id, require_empty)
+        @header = Header.new(hash_size: hash_size || 20, hash_id: hash_id || "sha1", file_id: file_id || random_id)
+        @store.write_at(0, @header.to_bytes + ("\0".b * 17))
+        publish(Slot.new, 0)
+      rescue Exception # rubocop:disable Lint/RescueException -- Cleanup must also preserve nonstandard exceptions.
+        reset_failed_initialization(require_empty)
+        raise
+      end
+
+      def reset_failed_initialization(require_empty)
+        # The exclusive initialization lock still owns this empty target.
+        @store.truncate(0) if require_empty && !@needs_reopen
+      rescue Exception # rubocop:disable Lint/RescueException -- Cleanup must also preserve nonstandard exceptions.
+        nil
+      end
+
+      def check_header(name, expected, actual)
+        return if expected.nil? || expected == actual
+
+        raise ProfileMismatchError, "#{name} does not match the database header"
+      end
+
+      def close_unreturned
+        @store.send(:close_unreturned)
+      end
+
+      def restore_empty_target
+        return if needs_reopen?
+
+        @store.with_exclusive_lock(timeout: @lock_timeout) do
+          current = Header.parse(@store.read_at(0, Header::SIZE))
+          next unless current.file_id == header.file_id
+
+          size, slot = committed_state
+          @store.truncate(0) if size == MINIMUM_SIZE && slot.empty?
+        end
+      rescue Exception # rubocop:disable Lint/RescueException -- Cleanup must also preserve nonstandard exceptions.
+        # Cleanup must preserve the original failure or nonlocal exit. A new
+        # generation or committed root belongs to its writer and is retained.
+        nil
+      end
+
+      protected :restore_empty_target
+
+      def active_transaction
+        transaction = @transaction
+        raise TransactionStateError, "allocation requires a transaction" unless transaction
+
+        transaction.check_active!
+        transaction
+      end
+
+      def check_range!(offset, size, bound)
+        unless offset.is_a?(Integer) && size.is_a?(Integer) && bound.is_a?(Integer) &&
+               offset >= DATA_OFFSET && size >= 0 && offset + size <= bound
+          raise CorruptionError, "object exceeds its read boundary"
+        end
+      end
+
+      def committed_state
+        check_open!
+        size = Format.unsigned(@store.read_at(ROOT_OFFSET + Slot::SIZE, 8))
+        raise CorruptionError, "invalid committed file size" if size < MINIMUM_SIZE || size > @store.length
+
+        slot, = Replication.trailer(@store, header, size, size)
+        [size, slot]
+      rescue EOFError => e
+        raise CorruptionError, e.message
+      end
+
+      def publish(slot, commit_id)
+        # @type var outcome: :not_committed | :committed | :unknown
+        outcome = :not_committed
+        position = @store.length
+        body = slot.to_bytes + [commit_id].pack("Q>")
+        checksum = CRC64.call([header.file_id, position].pack("Q>Q>") + body)
+        @store.append(body + [checksum].pack("Q>"))
+        @store.sync
+        outcome = :unknown
+        @store.write_at(ROOT_OFFSET, slot.to_bytes + [position + TRAILER_SIZE].pack("Q>"))
+        @store.sync
+      rescue Exception => e # rubocop:disable Lint/RescueException -- Cleanup must also preserve nonstandard exceptions.
+        @needs_reopen = true if outcome == :unknown
+        raise CommitError.new(e.message, outcome: outcome || :not_committed), cause: e
+      end
+
+      def random_id
+        Format.unsigned(SecureRandom.random_bytes(8))
+      end
+    end
+  end
+end

data/lib/xitdb/core/file_store.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+module XitDB
+  module Core
+    # A synchronized file store with bounded write buffering.
+    class FileStore
+      include Store
+
+      attr_reader :storage_identity
+
+      WRITERS = {} # rubocop:disable Style/MutableConstant -- Writer registry updates are protected by WRITERS_MUTEX.
+      WRITERS_MUTEX = Mutex.new
+      private_constant :WRITERS, :WRITERS_MUTEX
+
+      def self.open(path, readonly: false, buffer_limit: 2 * 1024 * 1024, create: true, exclusive: false)
+        raise TypeError, "buffer_limit must be an Integer" unless buffer_limit.is_a?(Integer)
+        raise ArgumentError, "buffer_limit must be positive" unless buffer_limit.positive?
+        raise ArgumentError, "exclusive opening requires writable creation" if exclusive && (readonly || !create)
+
+        file = open_file(path, readonly, create, exclusive)
+        begin
+          directory = File.dirname(File.expand_path(path)) unless readonly
+          new(file, readonly, buffer_limit, sync_directory: directory)
+        rescue Exception # rubocop:disable Lint/RescueException -- Cleanup must also preserve nonstandard exceptions.
+          file.close
+          raise
+        end
+      end
+
+      def self.open_file(path, readonly, create, exclusive)
+        flags = readonly ? File::RDONLY : File::RDWR
+        return File.open(path, flags) unless create && !readonly
+
+        begin
+          File.open(path, flags | File::CREAT | File::EXCL, 0o666)
+        rescue Errno::EEXIST
+          raise if exclusive
+
+          File.open(path, flags)
+        end
+      end
+      private_class_method :open_file
+
+      def initialize(file, readonly, buffer_limit, sync_directory: nil)
+        @file = file.binmode
+        @sync_directory = sync_directory
+        @readonly = readonly
+        @buffer_limit = buffer_limit
+        @pending = "".b
+        @pending_offset = 0
+        stat = @file.stat
+        @identity = [Process.pid, stat.dev, stat.ino].freeze
+        @storage_identity = @identity
+        writer = WRITERS_MUTEX.synchronize do
+          entry = WRITERS[@identity] ||= [Mutex.new, 0]
+          entry[1] += 1
+          entry[0]
+        end
+        initialize_state(writer)
+      end
+
+      def readonly?
+        @readonly
+      end
+
+      private
+
+      def store_length
+        disk_length = @file.stat.size
+        @pending.empty? ? disk_length : [disk_length, @pending_offset + @pending.bytesize].max
+      end
+
+      def read_bytes(offset, length)
+        return read_file(offset, length) if @pending.empty?
+
+        result = "".b
+        finish = offset + length
+        if offset < @pending_offset
+          count = [finish, @pending_offset].min - offset
+          disk_count = (@file.stat.size - offset).clamp(0, count)
+          result << read_file(offset, disk_count)
+          result << ("\0".b * (count - disk_count))
+          offset += count
+        end
+        if offset < finish && offset < @pending_offset + @pending.bytesize
+          count = [finish - offset, @pending_offset + @pending.bytesize - offset].min
+          result << Format.slice(@pending, offset - @pending_offset, count)
+          offset += count
+        end
+        result << read_file(offset, finish - offset) if offset < finish
+        result
+      end
+
+      def read_file(offset, length)
+        result = "".b
+        while result.bytesize < length
+          bytes = retry_interrupted { @file.pread(length - result.bytesize, offset + result.bytesize) }
+          raise EOFError, "read exceeds file length" if bytes.empty?
+
+          result << bytes
+        end
+        result
+      end
+
+      def write_bytes(offset, bytes)
+        if bytes.bytesize > @buffer_limit
+          flush_bytes
+          write_file(offset, bytes)
+          return
+        end
+
+        unless @pending.empty? || (offset.between?(@pending_offset, @pending_offset + @pending.bytesize) &&
+               offset + bytes.bytesize <= @pending_offset + @buffer_limit)
+          flush_bytes
+        end
+        @pending_offset = offset if @pending.empty?
+        @pending[offset - @pending_offset, bytes.bytesize] = bytes
+      end
+
+      def write_file(offset, bytes)
+        written = 0
+        while written < bytes.bytesize
+          count = retry_interrupted { @file.pwrite(Format.slice(bytes, written..), offset + written) }
+          raise IOError, "file write made no progress" unless count.positive?
+
+          written += count
+        end
+      end
+
+      def truncate_bytes(length)
+        retry_interrupted { @file.truncate(length) }
+        if length <= @pending_offset
+          @pending.clear
+        elsif length < @pending_offset + @pending.bytesize
+          @pending = Format.slice(@pending, 0, length - @pending_offset)
+        end
+      end
+
+      def flush_bytes
+        return if @pending.empty?
+
+        write_file(@pending_offset, @pending)
+        @pending.clear
+        nil
+      end
+
+      def sync_bytes
+        retry_interrupted { @file.fsync }
+        sync_directory = @sync_directory
+        if sync_directory
+          File.open(sync_directory, File::RDONLY) do |directory|
+            retry_interrupted { directory.fsync }
+          end
+          @sync_directory = nil
+        end
+        nil
+      end
+
+      def close_bytes
+        @file.close
+        WRITERS_MUTEX.synchronize do
+          entry = WRITERS.fetch(@identity)
+          entry[1] -= 1
+          WRITERS.delete(@identity) if entry[1].zero?
+        end
+      end
+
+      def lock_writer(deadline)
+        raise TransactionStateError, "writer lock for this file is already held by this thread" if @writer_mutex.owned?
+
+        super
+        acquired = false
+        begin
+          wait_for_writer(deadline) until retry_interrupted { @file.flock(File::LOCK_EX | File::LOCK_NB) }
+          acquired = true
+        ensure
+          @writer_mutex.unlock unless acquired
+        end
+      end
+
+      def unlock_writer
+        retry_interrupted { @file.flock(File::LOCK_UN) }
+      ensure
+        super
+      end
+
+      def retry_interrupted
+        yield
+      rescue Errno::EINTR
+        retry
+      end
+    end
+  end
+end

data/lib/xitdb/core/format.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+module XitDB
+  module Core
+    # Validates and decodes the binary storage format.
+    module Format
+      UINT64_MAX = (1 << 64) - 1
+
+      def self.integer(value, range)
+        raise TypeError, "expected Integer, got #{value.class}" unless value.is_a?(Integer)
+        raise RangeError, "integer outside #{range}" unless range.cover?(value)
+
+        value
+      end
+
+      def self.width(bytes, width)
+        raise TypeError, "expected String, got #{bytes.class}" unless bytes.is_a?(String)
+        raise CorruptionError, "expected #{width} bytes, got #{bytes.bytesize}" unless bytes.bytesize == width
+      end
+
+      def self.slice(bytes, offset, length = nil)
+        result = if offset.is_a?(Range)
+                   bytes.byteslice(offset)
+                 else
+                   bytes.byteslice(offset, length || 1)
+                 end
+        raise CorruptionError, "byte slice exceeds payload" unless result
+
+        result
+      end
+
+      def self.unsigned(bytes)
+        value = bytes.unpack1("Q>")
+        raise CorruptionError, "invalid unsigned integer" unless value.is_a?(Integer)
+
+        value
+      end
+
+      def self.integers(bytes, format)
+        bytes.unpack(format).map do |value|
+          raise CorruptionError, "invalid integer payload" unless value.is_a?(Integer)
+
+          value
+        end
+      end
+
+      def self.pair(bytes, format)
+        values = integers(bytes, format)
+        [values.fetch(0), values.fetch(1)]
+      end
+    end
+
+    # The immutable database format header.
+    class Header
+      SIZE = 20
+      VERSION = 1
+
+      attr_reader :file_id, :hash_id, :hash_size
+
+      def initialize(file_id:, hash_id:, hash_size:, padding: 0)
+        @file_id = Format.integer(file_id, 0..Format::UINT64_MAX)
+        @hash_size = Format.integer(hash_size, 0..65_535)
+        raise TypeError, "hash_id must be a String" unless hash_id.is_a?(String)
+        raise ArgumentError, "hash_id must contain four bytes" unless hash_id.bytesize == 4
+
+        @hash_id = hash_id.b.freeze
+        @padding = Format.integer(padding, 0..255)
+        freeze
+      end
+
+      def self.parse(bytes)
+        Format.width(bytes, SIZE)
+        magic = Format.slice(bytes, 0, 3)
+        padding = bytes.getbyte(3) || raise(CorruptionError, "missing header padding")
+        version, hash_size = Format.pair(Format.slice(bytes, 4, 4), "nn")
+        hash_id = Format.slice(bytes, 8, 4)
+        file_id = Format.unsigned(Format.slice(bytes, 12, 8))
+        raise CorruptionError, "invalid xitdb magic number" unless magic == "xit"
+        raise FormatVersionError, "unsupported xitdb format version #{version}" unless version == VERSION
+
+        new(file_id: file_id, hash_id: hash_id, hash_size: hash_size, padding: padding)
+      end
+
+      def to_bytes
+        ["xit", @padding, VERSION, hash_size, hash_id, file_id].pack("a3Cnn a4Q>")
+      end
+    end
+
+    # An immutable tagged value or record reference.
+    class Slot
+      SIZE = 9
+      TAGS = %i[none index array_list linked_array_list hash_map kv_pair bytes short_bytes uint int float
+                hash_set counted_hash_map counted_hash_set sorted_map sorted_set sorted_kv_pair].freeze
+
+      attr_reader :tag, :value, :full
+
+      def initialize(tag: :none, value: 0, full: false)
+        code = tag.is_a?(Symbol) ? TAGS.index(tag) : tag
+        raise ArgumentError, "invalid slot tag: #{tag.inspect}" unless code.is_a?(Integer) && (0..127).cover?(code)
+        raise TypeError, "full must be true or false" unless full.equal?(true) || full.equal?(false)
+
+        @tag = TAGS[code] || code
+        @value = Format.integer(value, 0..Format::UINT64_MAX)
+        @full = full
+        freeze
+      end
+
+      def self.parse(bytes)
+        Format.width(bytes, SIZE)
+        flags = bytes.getbyte(0) || raise(CorruptionError, "missing slot flags")
+        value = Format.unsigned(Format.slice(bytes, 1, 8))
+        new(tag: flags & 0x7f, value: value, full: flags.anybits?(0x80))
+      end
+
+      def self.int(value)
+        Format.integer(value, -(1 << 63)..((1 << 63) - 1))
+        new(tag: :int, value: value & Format::UINT64_MAX)
+      end
+
+      def self.uint(value)
+        new(tag: :uint, value: value)
+      end
+
+      def self.float(value)
+        raise TypeError, "expected Float, got #{value.class}" unless value.is_a?(Float)
+
+        new(tag: :float, value: Format.unsigned([value].pack("G")))
+      end
+
+      def self.null
+        new(full: true)
+      end
+
+      def empty?
+        tag == :none && !full
+      end
+
+      def ==(other)
+        other.is_a?(Slot) && tag == other.tag && full == other.full && value == other.value
+      end
+
+      def to_bytes
+        code = tag.is_a?(Symbol) ? TAGS.index(tag) : tag
+        [code | (full ? 0x80 : 0), value].pack("CQ>")
+      end
+    end
+  end
+end