rubydb 0.1.0

data/lib/rubydb/wal/writer.rb

@@ -0,0 +1,236 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module RubyDB
+  module WAL
+    # Writer - Writes records to the WAL
+    class Writer
+      attr_reader :current_segment, :current_lsn, :stats
+
+      def initialize(wal_dir, config = {})
+        @wal_dir = wal_dir
+        @config = config
+        @segment_max_size = config[:segment_size] || Segment::DEFAULT_MAX_SIZE
+        @current_segment = nil
+        @current_lsn = LSN.new(1, 0)
+        @buffer = []
+        @buffer_size = config[:buffer_size] || 1024 * 1024  # 1MB
+        @buffer_current_size = 0
+        @sync_on_write = config.key?(:sync) ? config[:sync] : true
+        @stats = {
+          records_written: 0,
+          bytes_written: 0,
+          segments_created: 0,
+          segments_rotated: 0,
+          buffer_flushes: 0,
+          syncs: 0
+        }
+        @lock = Mutex.new
+        @write_thread = nil
+        @running = false
+        @shutdown = false
+        @background_error = nil
+
+        # Create WAL directory
+        FileUtils.mkdir_p(@wal_dir)
+
+        # Create first segment
+        create_new_segment
+
+        # Start background writer if async
+        if config[:async] != false
+          start_background_writer
+        end
+      end
+
+      def write_record(record)
+        @lock.synchronize do
+          raise @background_error if @background_error
+          # Assign LSN
+          lsn = next_lsn
+          record.instance_variable_set(:@lsn, lsn)
+
+          # Serialize record
+          serialized = record.serialize
+          record_data = serialized[:checksum] + serialized[:data]
+
+          # Add to buffer
+          @buffer << { record: record, data: record_data, lsn: lsn }
+          @buffer_current_size += record_data.bytesize
+
+          @stats[:records_written] += 1
+
+          # Flush if buffer is full OR if sync is enabled (for durability)
+          if @buffer_current_size >= @buffer_size || @sync_on_write
+            flush_buffer
+            _sync if @sync_on_write
+          end
+
+          lsn
+        end
+      end
+
+      def write_batch(records)
+        @lock.synchronize do
+          start_lsn = nil
+          last_lsn = nil
+
+          records.each do |record|
+            lsn = write_record_locked(record)
+            start_lsn ||= lsn
+            last_lsn = lsn
+          end
+
+          flush_buffer if @buffer.any?
+
+          { start_lsn: start_lsn, end_lsn: last_lsn, count: records.size }
+        end
+      end
+
+      def flush
+        @lock.synchronize do
+          raise @background_error if @background_error
+          flush_buffer
+          _sync if @sync_on_write
+        end
+      end
+
+      def sync
+        @lock.synchronize do
+          _sync
+        end
+      end
+
+      private def _sync
+        if @current_segment && @current_segment.open?
+          @current_segment.instance_variable_get(:@file).fsync
+          @stats[:syncs] += 1
+        end
+      end
+
+      def shutdown(wait = true)
+        write_thread = nil
+        @lock.synchronize do
+          return if @shutdown
+
+          @shutdown = true
+          @running = false
+          @write_thread&.kill if !wait
+          flush_buffer
+          _sync if @sync_on_write
+          @current_segment.close if @current_segment&.open?
+          write_thread = @write_thread
+          @write_thread = nil
+        end
+
+        # The async writer can be asleep or flushing while shutdown begins.
+        # Join it before the caller removes the WAL directory; otherwise a
+        # late background flush can race segment close and leave Windows file
+        # handles open during temporary-directory cleanup.
+        write_thread&.join if wait && write_thread != Thread.current
+      end
+
+      def stats
+        @lock.synchronize do
+          @stats.merge({
+            buffer_size: @buffer.size,
+            buffer_bytes: @buffer_current_size,
+            current_segment: @current_segment&.segment_id,
+            current_lsn: @current_lsn.to_s,
+            async: @running,
+            last_background_error: @background_error&.message
+          })
+        end
+      end
+
+      private
+
+      def write_record_locked(record)
+        lsn = next_lsn
+        record.instance_variable_set(:@lsn, lsn)
+
+        serialized = record.serialize
+        record_data = serialized[:checksum] + serialized[:data]
+        @buffer << { record: record, data: record_data, lsn: lsn }
+        @buffer_current_size += record_data.bytesize
+        @stats[:records_written] += 1
+
+        if @buffer_current_size >= @buffer_size || @sync_on_write
+          flush_buffer
+          _sync if @sync_on_write
+        end
+
+        lsn
+      end
+
+      def create_new_segment
+        segment_id = @current_segment ? @current_segment.segment_id + 1 : 1
+        segment = Segment.new(segment_id, @wal_dir, @segment_max_size)
+        segment.open
+        @current_segment = segment
+        @stats[:segments_created] += 1
+        segment
+      end
+
+      def next_lsn
+        @current_lsn = LSN.new(
+          @current_segment.segment_id,
+          @current_segment.size + @buffer.sum { |entry| entry[:data].bytesize + Segment::FRAME_HEADER_SIZE }
+        )
+      end
+
+      def flush_buffer
+        return if @buffer.empty?
+
+        @stats[:buffer_flushes] += 1
+
+        @buffer.each do |entry|
+          # Check if current segment is full
+          if @current_segment && @current_segment.full?
+            rotate_segment
+          end
+
+          # Write to segment
+          result = @current_segment.append(entry[:data], entry[:lsn])
+          @stats[:bytes_written] += entry[:data].bytesize
+
+          if result == :full
+            rotate_segment
+            # Re-write to new segment
+            @current_segment.append(entry[:data], entry[:lsn])
+          end
+        end
+
+        @buffer.clear
+        @buffer_current_size = 0
+      end
+
+      def rotate_segment
+        @current_segment.close
+        @stats[:segments_rotated] += 1
+        create_new_segment
+      end
+
+      def start_background_writer
+        @running = true
+        @write_thread = Thread.new do
+          while !@shutdown
+            sleep(1)
+            begin
+              if @buffer.any?
+                flush
+              end
+            rescue => e
+              @lock.synchronize do
+                @background_error ||= e
+                @running = false
+              end
+              break
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rubydb.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+# Top-level entry point for the RubyDB library.
+# The project stores the implementation under lib/rubydb/rubydb.rb, so this wrapper
+# keeps the public require path compatible with typical Ruby package usage.
+
+require_relative "rubydb/rubydb"

data/packaging/docker/Dockerfile

@@ -0,0 +1,36 @@
+# syntax=docker/dockerfile:1
+
+FROM ruby:3.3-slim
+
+ENV APP_HOME=/opt/rubydb \
+    RUBYDB_DATA_DIR=/var/lib/rubydb/data \
+    RUBYDB_LOG_DIR=/var/log/rubydb \
+    RUBYDB_PID_FILE=/run/rubydb.pid \
+    RUBYDB_HOST=0.0.0.0 \
+    RUBYDB_PORT=7432
+
+WORKDIR ${APP_HOME}
+
+COPY Gemfile Gemfile.lock rubydb.gemspec ./
+COPY lib ./lib
+COPY exe ./exe
+COPY bin ./bin
+
+RUN bundle config set --without development:test \
+    && bundle install --jobs 4 --retry 3 \
+    && gem build rubydb.gemspec \
+    && gem install --no-document rubydb-*.gem \
+    && rm -f rubydb-*.gem \
+    && useradd --system --home-dir ${APP_HOME} --shell /usr/sbin/nologin rubydb \
+    && install -d -o rubydb -g rubydb ${RUBYDB_DATA_DIR} ${RUBYDB_LOG_DIR} /run/rubydb
+
+COPY packaging/docker/entrypoint.sh /usr/local/bin/rubydb-entrypoint
+RUN chmod 0755 /usr/local/bin/rubydb-entrypoint
+
+USER rubydb
+EXPOSE 7432
+VOLUME ["/var/lib/rubydb"]
+HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
+  CMD rubydb status --json >/dev/null || exit 1
+
+ENTRYPOINT ["/usr/local/bin/rubydb-entrypoint"]

data/packaging/docker/docker-compose.failover.yml

@@ -0,0 +1,43 @@
+services:
+  primary:
+    build:
+      context: ../..
+      dockerfile: packaging/docker/Dockerfile
+    hostname: rubydb-primary
+    environment:
+      RUBYDB_DATA_DIR: /var/lib/rubydb/data
+      RUBYDB_HOST: 0.0.0.0
+      RUBYDB_PORT: 7432
+    volumes:
+      - primary-data:/var/lib/rubydb
+    networks: [rubydb-cluster]
+
+  replica:
+    build:
+      context: ../..
+      dockerfile: packaging/docker/Dockerfile
+    hostname: rubydb-replica
+    environment:
+      RUBYDB_DATA_DIR: /var/lib/rubydb/data
+      RUBYDB_HOST: 0.0.0.0
+      RUBYDB_PORT: 7432
+    volumes:
+      - replica-data:/var/lib/rubydb
+    networks: [rubydb-cluster]
+
+  observer:
+    image: ruby:3.3-slim
+    working_dir: /workspace
+    volumes:
+      - ../..:/workspace:ro
+    command: ["ruby", "scripts/replication_network_failover_drill"]
+    networks: [rubydb-cluster]
+    profiles: [drill]
+
+networks:
+  rubydb-cluster:
+    driver: bridge
+
+volumes:
+  primary-data:
+  replica-data:

data/packaging/docker/entrypoint.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+set -eu
+
+: "${RUBYDB_DATA_DIR:=/var/lib/rubydb/data}"
+: "${RUBYDB_LOG_DIR:=/var/log/rubydb}"
+: "${RUBYDB_PID_FILE:=/run/rubydb.pid}"
+: "${RUBYDB_HOST:=0.0.0.0}"
+: "${RUBYDB_PORT:=7432}"
+
+exec rubydb start \
+  --host "$RUBYDB_HOST" \
+  --port "$RUBYDB_PORT" \
+  --data-dir "$RUBYDB_DATA_DIR" \
+  --log-dir "$RUBYDB_LOG_DIR" \
+  --pid-file "$RUBYDB_PID_FILE"

data/packaging/homebrew/rubydb.rb

@@ -0,0 +1,19 @@
+class Rubydb < Formula
+  desc "Developer-first relational database for Ruby"
+  homepage "https://github.com/aldanedev-create/rubydb"
+  url "https://github.com/aldanedev-create/rubydb/archive/refs/tags/v0.1.0.tar.gz"
+  version "0.1.0"
+  license "MIT"
+
+  depends_on "ruby@3.3"
+
+  def install
+    system "gem", "build", "rubydb.gemspec"
+    system "gem", "install", "--no-document", *Dir["rubydb-*.gem"]
+    bin.install "exe/rubydb"
+  end
+
+  test do
+    assert_match version.to_s, shell_output("#{bin}/rubydb --version")
+  end
+end

data/packaging/systemd/rubydb.service

@@ -0,0 +1,22 @@
+[Unit]
+Description=RubyDB database server
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=rubydb
+Group=rubydb
+ExecStart=/usr/local/bin/rubydb start --host 127.0.0.1 --port 7432 --data-dir /var/lib/rubydb/data --log-dir /var/log/rubydb --pid-file /run/rubydb.pid
+Restart=on-failure
+RestartSec=5s
+TimeoutStopSec=30s
+UMask=0077
+NoNewPrivileges=true
+PrivateTmp=true
+ProtectSystem=strict
+ProtectHome=true
+ReadWritePaths=/var/lib/rubydb /var/log/rubydb /run
+
+[Install]
+WantedBy=multi-user.target

data/rubydb.gemspec

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require_relative 'lib/rubydb/version'
+
+Gem::Specification.new do |spec|
+  spec.name = 'rubydb'
+  spec.version = RubyDB::VERSION
+  spec.authors = ['Aldane Hutchinson']
+  spec.email = ['aldanehutchinson5@gmail.com']
+
+  spec.summary = 'A developer-first relational database for Ruby'
+  spec.description = <<~DESC
+    RubyDB is a developer-first relational database foundation written in Ruby.
+    It combines SQLite-like simplicity with a growing SQL, WAL, MVCC, server,
+    replication, and Rails integration surface. Production deployment is
+    limited to the capabilities and validation documented by the project.
+  DESC
+  spec.homepage = 'https://github.com/aldanedev-create/rubydb'
+  spec.license = 'MIT'
+  # The implementation is currently compatible with the workspace Ruby runtime used for
+  # local development and CI. This is a practical compatibility target until the
+  # project adds a narrower support matrix and explicit Ruby-version policy.
+  spec.required_ruby_version = '>= 3.3.0'
+
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = 'https://github.com/aldanedev-create/rubydb/tree/main'
+  spec.metadata['changelog_uri'] = 'https://github.com/aldanedev-create/rubydb/blob/main/CHANGELOG.md'
+
+  # RubyGems signing is opt-in for release builders. The private key and
+  # certificate are supplied through protected CI paths and never committed.
+  signing_key = ENV['RUBYDB_GEM_SIGNING_KEY']
+  certificate = ENV['RUBYDB_GEM_CERT']
+  if signing_key || certificate
+    raise 'RUBYDB_GEM_SIGNING_KEY and RUBYDB_GEM_CERT must be provided together' unless signing_key && certificate
+    raise "RubyGems signing key not found: #{signing_key}" unless File.file?(signing_key)
+    raise "RubyGems certificate not found: #{certificate}" unless File.file?(certificate)
+
+    spec.signing_key = signing_key
+    spec.cert_chain = [File.read(certificate)]
+  end
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      f.match(%r{^(test|spec|features|benchmarks|fuzz|chaos|examples)/})
+    end
+  end
+  spec.bindir = 'exe'
+  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'digest', '~> 3.1'
+  spec.add_dependency 'json', '~> 2.7'
+  spec.add_dependency 'date', '~> 3.3'
+  spec.add_dependency 'time', '~> 0.4'
+end

data/scripts/benchmark

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "rbconfig"
+
+root = File.expand_path("..", __dir__)
+exec RbConfig.ruby, File.join(root, "benchmarks", "concurrent_workload.rb"), *ARGV

data/scripts/durability_drill

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Runs the real crash, filesystem-fault, corruption, compaction, and restore
+# paths as one release gate. Physical quota filling is opt-in because filling a
+# developer's filesystem without an explicit safety flag would be dangerous.
+
+require "json"
+require "rbconfig"
+
+root = File.expand_path("..", __dir__)
+Dir.chdir(root)
+specs = %w[
+  spec/filesystem_fault_injection_spec.rb
+  spec/transaction_crash_recovery_spec.rb
+  spec/crash_recovery_spec.rb
+  spec/crash_recovery_wal_failure_spec.rb
+  spec/corruption_repair_spec.rb
+  spec/compaction_persistence_spec.rb
+  spec/backup_restore_spec.rb
+  spec/incremental_backup_spec.rb
+  spec/delta_restore_spec.rb
+]
+
+command = [RbConfig.ruby, "-Ilib", "-S", "rspec", *specs]
+started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+success = system(*command)
+elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at
+
+result = {
+  success: success,
+  checks: %w[filesystem_faults crash_recovery corruption_detection compaction backup_restore],
+  physical_quota: ENV["RUBYDB_DURABILITY_QUOTA_DIR"] ? "operator_configured" : "not_run",
+  elapsed_seconds: elapsed.round(3)
+}
+puts JSON.generate(result)
+exit(success ? 0 : 1)

data/scripts/fuzz

@@ -0,0 +1,63 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Run the repository's fuzzers as a bounded CI or local safety gate.
+# Invalid SQL and intentionally injected WAL corruption are expected inputs;
+# uncaught errors and reported crashes are not.
+
+require "tmpdir"
+
+root = File.expand_path("..", __dir__)
+$LOAD_PATH.unshift(File.join(root, "lib"))
+require "rubydb"
+require_relative "../fuzz/sql_parser"
+require_relative "../fuzz/wal"
+require_relative "../fuzz/storage"
+require_relative "../fuzz/transactions"
+require_relative "../fuzz/query_engine"
+
+iterations = Integer(ENV.fetch("RUBYDB_FUZZ_ITERATIONS", "100"), 10)
+abort "RUBYDB_FUZZ_ITERATIONS must be positive" unless iterations.positive?
+seed = Integer(ENV.fetch("RUBYDB_FUZZ_SEED", "20260909"), 10)
+
+def run_fuzzer(name)
+  stats = yield
+  crashes = stats[:crashes].to_i
+  errors = stats[:errors].to_i
+  abort "#{name} found #{crashes} crash(es) and #{errors} error(s)" if crashes.positive? || errors.positive?
+  puts "#{name}: #{stats.inspect}"
+end
+
+Dir.mktmpdir("rubydb-fuzz-") do |directory|
+  run_fuzzer("SQL parser") do
+    RubyDB::Fuzz::SQLParserFuzzer.new(max_iterations: iterations, seed: seed).fuzz
+  end
+
+  run_fuzzer("WAL") do
+    RubyDB::Fuzz::WALFuzzer.new(
+      max_iterations: iterations,
+      seed: seed,
+      test_dir: File.join(directory, "wal")
+    ).fuzz
+  end
+
+  run_fuzzer("storage") do
+    RubyDB::Fuzz::StorageFuzzer.new(
+      max_iterations: iterations,
+      seed: seed,
+      test_dir: File.join(directory, "storage")
+    ).fuzz
+  end
+
+  engine = RubyDB::Storage::Engine.new(File.join(directory, "engine.rdb"))
+  begin
+    run_fuzzer("transactions") do
+      RubyDB::Fuzz::TransactionsFuzzer.new(engine, max_iterations: iterations, seed: seed).fuzz
+    end
+    run_fuzzer("query engine") do
+      RubyDB::Fuzz::QueryEngineFuzzer.new(engine, max_iterations: iterations, seed: seed).fuzz
+    end
+  ensure
+    engine.close
+  end
+end

data/scripts/release

@@ -0,0 +1,50 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "digest"
+require "rbconfig"
+require "rubygems"
+
+root = File.expand_path("..", __dir__)
+Dir.chdir(root)
+
+abort "release metadata preflight failed" unless system(RbConfig.ruby, File.join(root, "scripts", "release_check"))
+
+release_descriptors = [
+  "rubydb.gemspec",
+  "adapters/activerecord/rubydb-activerecord.gemspec",
+  "packaging/homebrew/rubydb.rb"
+]
+placeholder_pattern = /yourusername|your name|you@example\.com|your\.email@example\.com/i
+placeholder_files = release_descriptors.select do |path|
+  File.exist?(path) && File.read(path).match?(placeholder_pattern)
+end
+unless placeholder_files.empty?
+  abort "Release metadata contains placeholders in: #{placeholder_files.join(', ')}. Set the real owner, contact, homepage, source, and changelog URLs before publishing."
+end
+abort "rake build:checksum failed" unless system("rake", "build:checksum")
+
+if ENV["RUBYDB_GEM_SIGNING_KEY"] || ENV["RUBYDB_GEM_CERT"]
+  abort "Both RUBYDB_GEM_SIGNING_KEY and RUBYDB_GEM_CERT are required for gem signing" unless ENV["RUBYDB_GEM_SIGNING_KEY"] && ENV["RUBYDB_GEM_CERT"]
+  abort "RubyGems signing key does not exist" unless File.file?(ENV["RUBYDB_GEM_SIGNING_KEY"])
+  abort "RubyGems signing certificate does not exist" unless File.file?(ENV["RUBYDB_GEM_CERT"])
+end
+
+gem_path = Dir["pkg/rubydb-*.gem"].max_by { |path| File.mtime(path) }
+abort "No built gem found" unless gem_path
+checksum_path = File.join("checksums", "#{File.basename(gem_path)}.sha512")
+expected = File.read(checksum_path).split.first
+actual = Digest::SHA512.file(gem_path).hexdigest
+abort "Checksum verification failed" unless expected == actual
+
+if ENV["RUBYDB_PUBLISH"] == "1"
+  api_key = ENV.fetch("GEM_HOST_API_KEY", "")
+  credentials_path = Gem.configuration.credentials_path
+  credentials_available = File.file?(credentials_path) && !File.zero?(credentials_path)
+  abort "Set GEM_HOST_API_KEY or configure RubyGems credentials at #{credentials_path}" if api_key.empty? && !credentials_available
+  abort "gem push failed" unless system("gem", "push", gem_path)
+  puts "Published #{gem_path}"
+else
+  puts "Release artifact verified: #{gem_path}"
+  puts "To publish: RUBYDB_PUBLISH=1 GEM_HOST_API_KEY=<RubyGems API key> ruby scripts/release"
+end

data/scripts/release_check

@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Fail-closed release metadata validation. This stays separate from gem
+# building so CI can prove that a tag has a reviewed changelog entry before
+# any artifact is published.
+
+require "pathname"
+
+root = Pathname.new(__dir__).parent
+$LOAD_PATH.unshift(root.join("lib").to_s)
+require "rubydb/version"
+
+version = RubyDB::VERSION
+github_ref = ENV.fetch("GITHUB_REF_NAME", "").strip
+requested_version = ENV.fetch("RUBYDB_RELEASE_VERSION", "").strip
+
+tag_version = if github_ref.start_with?("v")
+  github_ref.delete_prefix("v")
+end
+
+if !requested_version.empty? && tag_version && requested_version != tag_version
+  abort "RUBYDB_RELEASE_VERSION #{requested_version} does not match GITHUB_REF_NAME #{github_ref}"
+end
+
+release_version = requested_version.empty? ? tag_version : requested_version
+if release_version.nil? || release_version.empty?
+  if ENV["RUBYDB_PUBLISH"] == "1"
+    abort "Publishing requires GITHUB_REF_NAME=v#{version} or RUBYDB_RELEASE_VERSION=#{version}"
+  end
+
+  puts "Release preflight skipped: no release tag or explicit release version"
+  exit 0
+end
+
+abort "Release version #{release_version} does not match RubyDB::VERSION #{version}" unless release_version == version
+
+changelog_path = root.join("CHANGELOG.md")
+changelog = changelog_path.read
+header = /^##\s+#{Regexp.escape(release_version)}(?:\s|$)/
+abort "CHANGELOG.md has no entry for version #{release_version}" unless changelog.match?(header)
+
+puts "Release preflight passed for RubyDB #{release_version}"