xitdb 0.1.0

data/lib/xitdb/core/memory_store.rb

@@ -0,0 +1,277 @@
+# frozen_string_literal: true
+
+require "monitor"
+
+module XitDB
+  module Core
+    # Coordinates store lifetime and exclusive writers.
+    module Store
+      # Only for owned stores whose construction or maintenance failed.
+      def close_unreturned
+        close
+      rescue Exception # rubocop:disable Lint/RescueException -- Cleanup must also preserve nonstandard exceptions.
+        begin
+          close(discard: true)
+        rescue Exception # rubocop:disable Lint/RescueException -- Cleanup must also preserve nonstandard exceptions.
+          nil
+        end
+      end
+      private :close_unreturned
+
+      def read_at(offset, length)
+        synchronize do
+          position(offset)
+          position(length)
+          next "".b if length.zero?
+          raise EOFError, "read exceeds store length" if offset + length > store_length
+
+          read_bytes(offset, length)
+        end
+      end
+
+      def write_at(offset, bytes)
+        synchronize do
+          writable!
+          position(offset)
+          string(bytes)
+          write_bytes(offset, bytes.b) unless bytes.empty?
+          bytes.bytesize
+        end
+      end
+
+      def append(bytes)
+        synchronize do
+          writable!
+          offset = store_length
+          write_at(offset, bytes)
+          offset
+        end
+      end
+
+      def length
+        synchronize { store_length }
+      end
+
+      def empty?
+        length.zero?
+      end
+
+      def truncate(length)
+        synchronize do
+          writable!
+          position(length)
+          truncate_bytes(length)
+          nil
+        end
+      end
+
+      def flush
+        synchronize { flush_bytes }
+      end
+
+      def sync
+        synchronize do
+          flush_bytes
+          sync_bytes
+        end
+      end
+
+      # Pin the handle across all storage calls in one logical operation.
+      def with_operation
+        thread = Thread.current
+        synchronize { @operation_users[thread] = @operation_users.fetch(thread, 0) + 1 }
+        begin
+          yield
+        ensure
+          @state.synchronize do
+            @operation_users[thread] -= 1
+            @operation_users.delete(thread) if @operation_users[thread].zero?
+            @idle.broadcast
+          end
+        end
+      end
+
+      def with_exclusive_lock(timeout: 5, on_release_error: nil)
+        raise TypeError, "timeout must be a real Numeric" unless timeout.is_a?(Numeric) && timeout.real?
+        raise ArgumentError, "timeout must be finite and nonnegative" unless timeout.finite? && timeout >= 0
+
+        deadline = monotonic_time + timeout
+        synchronize do
+          writable!
+          raise TransactionStateError, "writer lock is already held by this thread" if @lock_owner == Thread.current
+
+          @lock_users += 1
+        end
+        locked = false
+        completed = false
+        begin
+          lock_writer(deadline)
+          locked = true
+          @state.synchronize do
+            writable!
+            @lock_owner = Thread.current
+          end
+          result = yield
+          completed = true
+          result
+        ensure
+          begin
+            release_writer(locked, completed, on_release_error)
+          ensure
+            @state.synchronize do
+              @lock_users -= 1
+              @idle.broadcast
+            end
+          end
+        end
+      end
+
+      def close(discard: false)
+        @state.synchronize do
+          return if @closed
+          raise TransactionStateError, "cannot close inside the writer block" if @lock_owner == Thread.current
+          if @operation_users.key?(Thread.current)
+            raise TransactionStateError, "cannot close inside an active operation"
+          end
+
+          @closing = true
+          begin
+            @idle.wait until @lock_users.zero? && @operation_users.empty?
+            return if @closed
+
+            flush_bytes unless discard
+            close_bytes
+            @closed = true
+          ensure
+            @closing = false
+          end
+        end
+      end
+
+      def closed?
+        @state.synchronize { @closed }
+      end
+
+      private
+
+      def release_writer(locked, completed, on_release_error)
+        @state.synchronize do
+          @lock_owner = nil if locked
+          begin
+            unlock_writer if locked
+          rescue Exception => e # rubocop:disable Lint/RescueException -- Cleanup must also preserve nonstandard exceptions.
+            @release_failed = true
+            on_release_error&.call(e)
+            # Raising here would replace a pending exception or nonlocal exit.
+            raise if completed
+          end
+        end
+      end
+
+      def initialize_state(writer_mutex = Mutex.new)
+        @state = Monitor.new
+        @idle = @state.new_cond
+        @writer_mutex = writer_mutex
+        @lock_users = 0
+        @operation_users = {}
+        @lock_owner = nil
+        @closed = false
+        @closing = false
+        @release_failed = false
+      end
+
+      def synchronize
+        @state.synchronize do
+          if @closed || (@closing && @lock_owner != Thread.current && !@operation_users.key?(Thread.current))
+            raise ClosedError, "store is closed"
+          end
+
+          yield
+        end
+      end
+
+      def writable!
+        raise ReadOnlyError, "store is read-only" if readonly?
+        raise TransactionStateError, "writer lock release failed; reopen the store" if @release_failed
+      end
+
+      def position(value)
+        raise TypeError, "offset and length must be Integers" unless value.is_a?(Integer)
+        raise ArgumentError, "offset and length must be nonnegative" if value.negative?
+      end
+
+      def string(bytes)
+        raise TypeError, "bytes must be a String" unless bytes.is_a?(String)
+      end
+
+      def monotonic_time
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+
+      def wait_for_writer(deadline)
+        remaining = deadline - monotonic_time
+        raise BusyError, "writer lock timed out" unless remaining.positive?
+
+        sleep([remaining, 0.005].min)
+      end
+
+      def lock_writer(deadline)
+        wait_for_writer(deadline) until @writer_mutex.try_lock
+      end
+
+      def unlock_writer
+        @writer_mutex.unlock
+      end
+    end
+    private_constant :Store
+
+    # A synchronized store backed by a binary string.
+    class MemoryStore
+      include Store
+
+      attr_reader :storage_identity
+
+      def initialize(bytes = "".b)
+        string(bytes)
+        @bytes = bytes.b
+        @storage_identity = Object.new.freeze
+        initialize_state
+      end
+
+      def readonly?
+        false
+      end
+
+      private
+
+      def store_length
+        @bytes.bytesize
+      end
+
+      def read_bytes(offset, length)
+        Format.slice(@bytes, offset, length)
+      end
+
+      def write_bytes(offset, bytes)
+        @bytes << ("\0".b * (offset - store_length)) if offset > store_length
+        @bytes[offset, bytes.bytesize] = bytes
+      end
+
+      def truncate_bytes(length)
+        if length < store_length
+          @bytes = Format.slice(@bytes, 0, length)
+        else
+          @bytes << ("\0".b * (length - store_length))
+        end
+      end
+
+      def flush_bytes; end
+
+      def sync_bytes; end
+
+      def close_bytes
+        @bytes.clear
+      end
+    end
+  end
+end

data/lib/xitdb/core/offset_map.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require "tempfile"
+
+module XitDB
+  module Core
+    # Tracks copied offsets and spills large maps to a temporary store.
+    class OffsetMap
+      CACHE_LIMIT = 16_384
+
+      attr_reader :peak_cache_size
+
+      def initialize
+        @cache = {}
+        @peak_cache_size = 0
+      end
+
+      def fetch(offset)
+        raise ClosedError, "offset map is closed" if @closed
+
+        map = @map
+        map ? map.get(key(offset))&.value : @cache[offset]
+      end
+
+      def store(offset, target)
+        raise ClosedError, "offset map is closed" if @closed
+
+        spill if !@map && @cache.size == CACHE_LIMIT && !@cache.key?(offset)
+        map = @map
+        if map
+          map.put(key(offset), key: Slot.new, value: Slot.uint(target))
+        else
+          @cache[offset] = target
+          @peak_cache_size = [@peak_cache_size, @cache.size].max
+        end
+        target
+      end
+
+      def close
+        return if @cleanup_complete
+
+        @closed = true
+        @cache.clear
+        @transaction&.expire!
+        begin
+          @store&.close(discard: true)
+        ensure
+          @temporary&.close!
+        end
+        @cleanup_complete = true
+        nil
+      end
+
+      private
+
+      def key(offset)
+        [offset].pack("Q>")
+      end
+
+      def spill
+        temporary = Tempfile.new("xitdb-offset-map-")
+        @temporary = temporary
+        store = FileStore.open(temporary.path || raise(IOError, "temporary file has no path"))
+        @store = store
+        database = Database.new(store: store, hash_size: 8, hash_id: "offs")
+        @database = database
+        # The scratch file is private and never published. One transaction keeps
+        # every mapping available without a commit and fsync for each insertion.
+        transaction = Transaction.new(database, store, Slot.new)
+        @transaction = transaction
+        map = HashMap.new(transaction.root)
+        @map = map
+        @cache.each { |offset, target| map.put(key(offset), key: Slot.new, value: Slot.uint(target)) }
+        @cache.clear
+      rescue Exception # rubocop:disable Lint/RescueException -- Cleanup must also preserve nonstandard exceptions.
+        begin
+          close
+        rescue Exception # rubocop:disable Lint/RescueException -- Cleanup must also preserve nonstandard exceptions.
+          nil
+        end
+        raise
+      end
+    end
+  end
+end

data/lib/xitdb/core/replication.rb

@@ -0,0 +1,242 @@
+# frozen_string_literal: true
+
+module XitDB
+  module Core
+    ReplicaState = Data.define(:committed_size, :checksum)
+
+    # Transfers committed byte ranges while retaining commit outcome semantics.
+    module Replication
+      CHUNK_SIZE = 64 * 1024
+
+      def replica_state
+        with_operation do
+          size, = committed_state
+          _, checksum = Replication.trailer(@store, header, size, size)
+          ReplicaState.new(committed_size: size, checksum: checksum)
+        end
+      end
+
+      def replication_start(state)
+        raise TypeError, "expected ReplicaState" unless state.is_a?(ReplicaState)
+
+        Replication.position(state.committed_size)
+        Format.integer(state.checksum, 0..Format::UINT64_MAX)
+        with_operation do
+          size, = committed_state
+          next 0 if state.committed_size > size || state.committed_size < Database::MINIMUM_SIZE
+
+          begin
+            _, checksum = Replication.trailer(@store, header, state.committed_size, size)
+            checksum == state.checksum ? state.committed_size : 0
+          rescue CorruptionError
+            0
+          end
+        end
+      end
+
+      def read_committed(start, finish, io)
+        Replication.position(start)
+        Replication.position(finish)
+        with_operation do
+          size, = committed_state
+          if start > finish || finish > size || (start.positive? && start < Database::DATA_OFFSET)
+            raise ArgumentError, "invalid committed byte range"
+          end
+
+          if start.zero?
+            slot, = Replication.trailer(@store, header, finish, size)
+            Replication.write_all(io, @store.read_at(0, Header::SIZE))
+            Replication.write_all(io, slot.to_bytes + [finish].pack("Q>"))
+            start = Database::DATA_OFFSET
+          end
+          while start < finish
+            chunk = @store.read_at(start, [finish - start, CHUNK_SIZE].min)
+            Replication.write_all(io, chunk)
+            start += chunk.bytesize
+          end
+        end
+        nil
+      end
+
+      def apply_replicated(start, io, length)
+        Replication.position(start)
+        Replication.position(length)
+        raise ArgumentError, "full replacement requires Database.copy_replica" if start.zero?
+        raise TransactionStateError, "database requires reopening" if @needs_reopen
+
+        # @type var outcome: :not_committed | :committed | :unknown
+        outcome = :not_committed
+        completed = false
+        @store.with_exclusive_lock(timeout: @lock_timeout, on_release_error: ->(_error) { @needs_reopen = true }) do
+          raise TransactionStateError, "database requires reopening" if @needs_reopen
+
+          size, = committed_state
+          raise ArgumentError, "replication start must equal committed size" unless start == size
+
+          begin
+            append_replica_chunks(io, length, size)
+            if length.positive?
+              finish = start + length
+              slot, = Replication.trailer(@store, header, finish, finish)
+              Replication.storage_write { @store.sync }
+              outcome = :unknown
+              Replication.storage_write(outcome: outcome) do
+                @store.write_at(Database::ROOT_OFFSET, slot.to_bytes + [finish].pack("Q>"))
+                @store.sync
+              end
+              outcome = :committed
+            end
+            completed = true
+          ensure
+            recover_replica_application(size, completed, outcome)
+          end
+        end
+        nil
+      rescue Exception => e # rubocop:disable Lint/RescueException -- Preserve durability reporting for interrupts and fatal errors.
+        # A release failure after successful application cannot undo durability.
+        if completed && !e.is_a?(CommitError)
+          raise CommitError.new(e.message, outcome: outcome || :not_committed), cause: e
+        end
+
+        raise
+      end
+
+      def append_replica_chunks(io, length, size)
+        Replication.storage_write { @store.truncate(size) } if @store.length > size
+        Replication.read_chunks(io, length) do |chunk|
+          Replication.storage_write { @store.append(chunk) }
+        end
+      end
+
+      def recover_replica_application(size, completed, outcome)
+        @needs_reopen = true if outcome == :unknown
+        return if completed || outcome != :not_committed
+
+        begin
+          @store.truncate(size)
+        rescue Exception # rubocop:disable Lint/RescueException -- Rollback failure requires reopening even after an interrupt.
+          @needs_reopen = true
+        end
+      end
+      private :append_replica_chunks, :recover_replica_application
+
+      def self.copy_replica(to:, source_io:, length:)
+        position(length)
+        raise ArgumentError, "full replica is shorter than a database" if length < Database::MINIMUM_SIZE
+
+        store = FileStore.open(to)
+        # @type var outcome: :not_committed | :committed | :unknown
+        outcome = :not_committed
+        completed = false
+        begin
+          store.with_exclusive_lock do
+            # FileStore measures bytes and has no collection #empty? method.
+            unless store.length.zero? # rubocop:disable Style/ZeroLengthPredicate
+              raise ArgumentError, "replica staging target must be empty"
+            end
+
+            begin
+              read_chunks(source_io, length) { |chunk| storage_write { store.append(chunk) } }
+              validate_replica(store, length)
+
+              outcome = :unknown
+              storage_write(outcome: outcome) { store.sync }
+              outcome = :committed
+              completed = true
+            ensure
+              cleanup_staged_replica(store, completed, outcome)
+            end
+          end
+          database = Database.new(store: store, initialize_empty: false)
+          returned = true
+          database
+        rescue Exception => e # rubocop:disable Lint/RescueException -- Report durable commits even when cleanup is interrupted.
+          raise CommitError.new(e.message, outcome: outcome), cause: e if completed && !e.is_a?(CommitError)
+
+          raise
+        ensure
+          store.send(:close_unreturned) unless returned
+        end
+      end
+
+      def self.cleanup_staged_replica(store, completed, outcome)
+        return if completed || outcome != :not_committed
+
+        begin
+          store.truncate(0)
+        rescue Exception # rubocop:disable Lint/RescueException -- Preserve the original failure if staging cleanup fails.
+          nil
+        end
+      end
+      private_class_method :cleanup_staged_replica
+
+      def self.validate_replica(store, length)
+        header = Header.parse(store.read_at(0, Header::SIZE))
+        root = store.read_at(Database::ROOT_OFFSET, 17)
+        unless Format.unsigned(Format.slice(root, 9, 8)) == length
+          raise CorruptionError, "full replica length does not match its root header"
+        end
+
+        slot, = trailer(store, header, length, length)
+        return if root.byteslice(0, 9) == slot.to_bytes
+
+        raise CorruptionError, "full replica root does not match its trailer"
+      end
+      private_class_method :validate_replica
+
+      def self.position(value)
+        raise TypeError, "offset and length must be Integers" unless value.is_a?(Integer)
+        raise ArgumentError, "offset and length must be nonnegative" if value.negative?
+      end
+
+      def self.trailer(store, header, finish, bound)
+        raise CorruptionError, "invalid root trailer position" unless finish.between?(Database::MINIMUM_SIZE, bound)
+
+        position = finish - Database::TRAILER_SIZE
+        bytes = store.read_at(position, Database::TRAILER_SIZE)
+        slot = Slot.parse(Format.slice(bytes, 0, 9))
+        raise CorruptionError, "unknown root slot tag" unless Slot::TAGS.include?(slot.tag)
+
+        checksum = Format.unsigned(Format.slice(bytes, 17, 8))
+        expected = CRC64.call([header.file_id, position].pack("Q>Q>") + Format.slice(bytes, 0, 17))
+        raise CorruptionError, "invalid root trailer checksum" unless checksum == expected
+
+        [slot, checksum]
+      rescue EOFError => e
+        raise CorruptionError, e.message
+      end
+
+      def self.read_chunks(io, length)
+        while length.positive?
+          count = [length, CHUNK_SIZE].min
+          bytes = io.read(count)
+          raise EOFError, "replication stream ended before its declared length" if bytes.nil? || bytes == ""
+          unless bytes.is_a?(String) && bytes.bytesize <= count
+            raise IOError, "replication reader returned an invalid chunk"
+          end
+
+          yield bytes
+          length -= bytes.bytesize
+        end
+      end
+
+      def self.write_all(io, bytes)
+        offset = 0
+        while offset < bytes.bytesize
+          count = io.write(Format.slice(bytes, offset..))
+          unless count.is_a?(Integer) && count.positive? && count <= bytes.bytesize - offset
+            raise IOError, "replication writer made invalid progress"
+          end
+
+          offset += count
+        end
+      end
+
+      def self.storage_write(outcome: :not_committed)
+        yield
+      rescue Exception => e # rubocop:disable Lint/RescueException -- Every storage failure must retain its commit outcome.
+        raise CommitError.new(e.message, outcome: outcome), cause: e
+      end
+    end
+  end
+end