rubydb 0.1.5 → 0.1.6

data/lib/rubydb/execution/sort_executor.rb

@@ -97,11 +97,11 @@ module RubyDB
           comparison = 0
           order_by.each do |order|
             col = order.column.to_s
-            val_a = a[col]
-            val_b = b[col]
+            val_a = order_value(a, col)
+            val_b = order_value(b, col)
 
             comparison = compare_values(val_a, val_b)
-            comparison = -comparison if order.direction == :desc
+            comparison = -comparison if order.direction.to_s.casecmp?("desc")
             break unless comparison == 0
           end
           comparison
@@ -112,9 +112,11 @@ module RubyDB
         return 0 if a.nil? && b.nil?
         return -1 if a.nil?
         return 1 if b.nil?
-        if a.is_a?(String) && b.is_a?(String)
-        end
-        b
+
+        comparison = a <=> b
+        return comparison unless comparison.nil?
+
+        a.to_s <=> b.to_s
       end
 
       def external_sort(rows, order_by, temp_dir)
@@ -166,14 +168,26 @@ module RubyDB
       def compare_rows(a, b, order_by)
         order_by.each do |order|
           col = order.column.to_s
-          val_a = a[col]
-          val_b = b[col]
+          val_a = order_value(a, col)
+          val_b = order_value(b, col)
           comparison = compare_values(val_a, val_b)
+          comparison = -comparison if order.direction.to_s.casecmp?("desc")
           return comparison unless comparison == 0
         end
         0
       end
 
+      def order_value(row, column)
+        return row[column] if row.key?(column)
+        return row[column.to_sym] if row.key?(column.to_sym)
+
+        short_column = column.split(".").last
+        return row[short_column] if row.key?(short_column)
+        return row[short_column.to_sym] if row.key?(short_column.to_sym)
+
+        nil
+      end
+
       attr_reader :stats
     end
   end

data/lib/rubydb/indexes/btree.rb

@@ -133,6 +133,27 @@ module RubyDB
         })
       end
 
+      # Return a stable, ordered copy of the in-memory B-tree entries. Indexes
+      # are rebuilt from metadata on open, so this immutable value is the
+      # correct hand-off to a storage snapshot; it is not a second persisted
+      # index format.
+      def snapshot_entries
+        @lock.synchronize do
+          return [] unless @root
+
+          leaf = @root
+          leaf = leaf.children.first until leaf.is_leaf
+          entries = []
+          while leaf
+            leaf.keys.each_with_index do |key, index|
+              entries << {key: key, value: leaf.values[index]}
+            end
+            leaf = leaf.next_leaf
+          end
+          entries
+        end
+      end
+
       private
 
       def initialize_root
@@ -151,9 +172,17 @@ module RubyDB
       def extract_key(row)
         if @columns.size == 1
           column = @columns.first
-          row.key?(column) ? row[column] : row[column.to_s]
+          return row[column] if row.key?(column)
+          return row[column.to_s] if row.key?(column.to_s)
+
+          row[column.to_sym] if column.respond_to?(:to_sym)
         else
-          @columns.map { |col| row.key?(col) ? row[col] : row[col.to_s] }
+          @columns.map do |column|
+            next row[column] if row.key?(column)
+            next row[column.to_s] if row.key?(column.to_s)
+
+            column.respond_to?(:to_sym) ? row[column.to_sym] : nil
+          end
         end
       end
 

data/lib/rubydb/rubydb.rb

@@ -12,6 +12,7 @@ require "set"
 require_relative "version"
 require_relative "constants"
 require_relative "build_info"
+require_relative "accelerator"
 
 require_relative "errors/error"
 require_relative "errors/connection_error"
@@ -77,8 +78,12 @@ require_relative "sql/planner/binder"
 require_relative "sql/planner/analyzer"
 require_relative "sql/planner/type_checker"
 
-require_relative "execution/executor"
 require_relative "execution/plan"
+require_relative "execution/physical_plan"
+require_relative "execution/cost_model"
+require_relative "execution/operator_selection"
+require_relative "execution/accelerator_dispatch"
+require_relative "execution/executor"
 require_relative "execution/planner"
 require_relative "execution/optimizer"
 require_relative "execution/expression"
@@ -302,6 +307,7 @@ require_relative "cli/commands/diff"
 require_relative "cli/commands/inspect"
 require_relative "cli/commands/vacuum"
 require_relative "cli/commands/doctor"
+require_relative "cli/commands/accelerator"
 
 module RubyDB
   # Main database class

data/lib/rubydb/server/session.rb

@@ -80,6 +80,8 @@ module RubyDB
             case request[:type]
             when "query"
               process_query(request[:sql], request[:params] || [], request[:deadline_at], cancellation)
+            when "metadata"
+              process_metadata(request[:table])
             when "prepare"
               process_prepare(request[:sql])
             when "execute"
@@ -195,6 +197,49 @@ module RubyDB
         }
       end
 
+      def process_metadata(table_name = nil)
+        engine = @config[:engine]
+        raise RubyDB::ServerError, "Session has no database engine" unless engine
+
+        if table_name.nil? || table_name.to_s.empty?
+          return {
+            success: true,
+            type: "metadata_result",
+            metadata: {tables: engine.list_tables.map(&:to_s)},
+            timestamp: Time.now.iso8601
+          }
+        end
+
+        table_key = table_name.to_s
+        metadata = engine.table_metadata[table_key] || engine.table_metadata[table_key.to_sym]
+        raise RubyDB::DatabaseError, "Table '#{table_name}' does not exist" unless metadata
+
+        columns = engine.table_columns(table_key).map do |column|
+          {
+            name: column.name.to_s,
+            type: column.type.to_s,
+            nullable: column.nullable?,
+            primary_key: column.primary_key?,
+            default: column.has_default? ? column.default : nil
+          }
+        end
+        indexes = engine.index_manager.get_indexes_for_table(table_key).map do |index|
+          {name: index.name.to_s, unique: index.unique, columns: index.columns.map(&:to_s)}
+        end
+
+        {
+          success: true,
+          type: "metadata_result",
+          metadata: {
+            tables: [table_key],
+            columns: columns,
+            indexes: indexes,
+            constraints: metadata[:constraints] || []
+          },
+          timestamp: Time.now.iso8601
+        }
+      end
+
       def process_prepare(sql)
         permission_error = authorize_sql(sql)
         return permission_error if permission_error

data/lib/rubydb/storage/engine.rb

@@ -5,6 +5,7 @@ require "json"
 require "securerandom"
 require "fileutils"
 require "monitor"
+require_relative "../accelerator"
 
 # Storage components
 require_relative "database_lock"
@@ -17,6 +18,7 @@ require_relative "tuple"
 require_relative "serializer"
 require_relative "deserializer"
 require_relative "visibility_map"
+require_relative "snapshot_reader"
 require_relative "../mvcc/version_store"
 require_relative "storage_layout"
 require_relative "../indexes/index"
@@ -53,12 +55,13 @@ module RubyDB
       NULL_BITMAP_FLAG = 0x02
       VARIABLE_LENGTH_PREFIXES_FLAG = 0x04
 
-      attr_reader :storage_manager, :buffer_pool, :page_manager, :catalog, :path, :last_commit_ack
-      attr_reader :table_metadata, :transaction_manager, :wal, :crash_recovery, :version_store, :index_manager
+      attr_reader :storage_manager, :buffer_pool, :page_manager, :catalog, :path, :last_commit_ack, :config, :accelerator
+      attr_reader :table_metadata, :transaction_manager, :wal, :crash_recovery, :version_store, :index_manager, :snapshot_reader
 
       def initialize(path, config = {})
         @path = path
         @config = config
+        @accelerator = RubyDB::Accelerator::Client.new(config[:accelerator] || {})
         @database_lock = DatabaseLock.new(path)
         @database_lock.acquire!
         @catalog = config[:catalog] || Catalog::Catalog.new
@@ -117,7 +120,7 @@ module RubyDB
 
         # Initialize WAL
         wal_dir = config[:wal_dir] || "#{path}.wal"
-        @wal = WAL::WAL.new(wal_dir, recovery: false)  # Defer recovery until after metadata load
+        @wal = WAL::WAL.new(wal_dir, config.merge(recovery: false))  # Defer recovery until after metadata load
         @wal.attach_engine(self)
 
         # Initialize crash recovery
@@ -131,6 +134,7 @@ module RubyDB
 
         # Build/load indexes only after metadata and crash recovery are ready.
         @index_manager = Indexes::IndexManager.new(self)
+        @snapshot_reader = SnapshotReader.new(self)
 
         # Start cleanup thread if configured
         start_cleanup_thread if config[:auto_cleanup] != false
@@ -168,6 +172,31 @@ module RubyDB
         @storage_manager.free_page(page_number)
       end
 
+      # Yield a stable, immutable database snapshot to the accelerator. A
+      # transaction or an active MVCC writer makes the snapshot ineligible;
+      # callers use the normal Ruby path in that case.
+      def with_accelerator_snapshot
+        @lock.synchronize do
+          return nil unless accelerator_snapshot_eligible?
+
+          @storage_manager.flush
+          @visibility_map.flush
+          save_table_metadata
+          @snapshot_reader.with_snapshot { |snapshot| yield snapshot }
+        end
+      end
+
+      def accelerator_snapshot_hidden_row_ids
+        @visibility_map.visibility_info.each_with_object([]) do |(row_id, info), hidden|
+          state = (info[:state] || info["state"]).to_s.to_sym
+          hidden << row_id.to_i unless %i[visible committed].include?(state)
+        end
+      end
+
+      def table_pages_for_snapshot(table_name)
+        @table_pages[resolve_table_name(table_name)] || []
+      end
+
       # Record operations
       def read_record(page_number, offset, length)
         @storage_manager.read_record(page_number, offset, length)
@@ -1483,6 +1512,14 @@ module RubyDB
         current_transaction_manager && current_transaction_manager[:active]
       end
 
+      def accelerator_snapshot_eligible?
+        return false unless @is_open
+        return false if in_transaction?
+        return false unless @visibility_map.active_transaction_count.zero?
+
+        current_transaction_id.to_i.zero?
+      end
+
       def create_savepoint(name)
         @lock.synchronize do
           tx = current_transaction_manager
@@ -1613,6 +1650,8 @@ module RubyDB
         return unless @is_open
 
         stop_cleanup_thread
+        @accelerator&.close
+        @snapshot_reader&.close
 
         # Flush WAL first (ensures all mutations are recorded)
         if @wal
@@ -1653,6 +1692,7 @@ module RubyDB
             transaction_rollback: @stats[:transaction_rollback],
             wal_writes: @stats[:wal_writes],
             crash_recoveries: @stats[:crash_recoveries],
+            accelerator: @accelerator&.stats,
             last_maintenance_error: @last_maintenance_error,
             is_open: @is_open,
             visibility_rows: @visibility_map.visibility_info.size,
@@ -2092,7 +2132,7 @@ module RubyDB
       end
 
       def validate_referential_update!(table_name, old_row, new_row)
-        return true if old_row == new_row
+        return true if referential_row_values_equal?(table_name, old_row, new_row)
 
         @table_metadata.each do |child_table, child_metadata|
           (child_metadata[:constraints] || []).each do |raw_definition|
@@ -2102,13 +2142,13 @@ module RubyDB
 
             child_columns = Array(definition[:columns]).map(&:to_sym)
             parent_columns = Array(definition[:reference_columns] || :id).map(&:to_sym)
-            next unless parent_columns.any? { |column| old_row[column] != new_row[column] }
+            next unless parent_columns.any? { |column| row_value(old_row, column) != row_value(new_row, column) }
 
             child_rows = select_rows(child_table, child_metadata[:columns], visibility_check: false)
             referenced = child_rows.any? do |child_row|
               child_columns.each_with_index.all? do |child_column, index|
-                child_value = child_row[child_column] || child_row[child_column.to_s]
-                parent_value = old_row[parent_columns[index]] || old_row[parent_columns[index].to_s]
+                child_value = row_value(child_row, child_column)
+                parent_value = row_value(old_row, parent_columns[index])
                 !child_value.nil? && child_value == parent_value
               end
             end
@@ -2134,7 +2174,7 @@ module RubyDB
       end
 
       def apply_referential_update!(table_name, old_row, new_row)
-        return true if old_row == new_row
+        return true if referential_row_values_equal?(table_name, old_row, new_row)
 
         @table_metadata.each do |child_table, child_metadata|
           (child_metadata[:constraints] || []).each do |raw_definition|
@@ -2144,12 +2184,12 @@ module RubyDB
 
             child_columns = Array(definition[:columns]).map(&:to_sym)
             parent_columns = Array(definition[:reference_columns] || :id).map(&:to_sym)
-            next unless parent_columns.any? { |column| old_row[column] != new_row[column] }
+            next unless parent_columns.any? { |column| row_value(old_row, column) != row_value(new_row, column) }
             child_rows = select_rows(child_table, child_metadata[:columns], visibility_check: false)
             matching_rows = child_rows.select do |child_row|
               child_columns.each_with_index.all? do |child_column, index|
-                child_value = child_row[child_column] || child_row[child_column.to_s]
-                parent_value = old_row[parent_columns[index]] || old_row[parent_columns[index].to_s]
+                child_value = row_value(child_row, child_column)
+                parent_value = row_value(old_row, parent_columns[index])
                 !child_value.nil? && child_value == parent_value
               end
             end
@@ -2160,7 +2200,7 @@ module RubyDB
               child_row_id = child_row[:_row_id] || child_row["_row_id"]
               updates = case action
               when "cascade"
-                child_columns.each_with_index.to_h { |column, index| [column, new_row[parent_columns[index]] || new_row[parent_columns[index].to_s]] }
+                child_columns.each_with_index.to_h { |column, index| [column, row_value(new_row, parent_columns[index])] }
               when "set_null"
                 child_columns.to_h { |column| [column, nil] }
               when "set_default"
@@ -2178,6 +2218,28 @@ module RubyDB
         true
       end
 
+      # Rows can cross the storage, SQL, WAL, and recovery boundaries with
+      # symbol or string keys. Referential actions compare column values, not
+      # the Ruby key representation. Without this normalization, updating a
+      # non-key parent column could look like a primary-key change during WAL
+      # replay and incorrectly fail ON UPDATE RESTRICT.
+      def row_value(row, column)
+        return row[column] if row.key?(column)
+        return row[column.to_s] if row.key?(column.to_s)
+
+        nil
+      end
+
+      def referential_row_values_equal?(table_name, old_row, new_row)
+        columns = @table_metadata[table_name.to_s] || @table_metadata[table_name.to_sym]
+        column_definitions = columns && columns[:columns]
+        return old_row == new_row if column_definitions.nil?
+
+        column_definitions.all? do |column|
+          row_value(old_row, column.name.to_sym) == row_value(new_row, column.name.to_sym)
+        end
+      end
+
       def cache_hit_rate
         total = @stats[:cache_hits] + @stats[:cache_misses]
         return 0.0 if total == 0

data/lib/rubydb/storage/snapshot_reader.rb

@@ -0,0 +1,167 @@
+# frozen_string_literal: true
+
+require "digest"
+require "fileutils"
+require "json"
+require "securerandom"
+
+module RubyDB
+  module Storage
+    # Creates a validated snapshot manifest for the Go accelerator. The
+    # default is a lock-protected read of the already-flushed live file;
+    # operators can opt into a detached copy when they need one. Ruby remains
+    # responsible for locking, WAL durability, catalog state, and MVCC
+    # eligibility.
+    class SnapshotReader
+      FORMAT_VERSION = 1
+
+      def initialize(engine)
+        @engine = engine
+        @directory = "#{engine.path}.accelerator-snapshots"
+        @lock = Mutex.new
+        @active_paths = {}
+      end
+
+      def with_snapshot
+        snapshot = create
+        begin
+          yield snapshot
+        ensure
+          release(snapshot)
+        end
+      end
+
+      def close
+        @lock.synchronize do
+          @active_paths.keys.each { |path| delete_file(path) }
+          @active_paths.clear
+          Dir.rmdir(@directory) if Dir.exist?(@directory) && Dir.empty?(@directory)
+        end
+      end
+
+      private
+
+      def create
+        source = File.expand_path(@engine.path)
+        raise StorageError, "Cannot create accelerator snapshot: database file is missing" unless File.file?(source)
+
+        # Engine#with_accelerator_snapshot holds the engine lock for the whole
+        # yield, including the Go read. In that guarded section the live file
+        # is already flushed and no Ruby writer can mutate it, so copying the
+        # complete database before every read only adds latency and doubles
+        # the storage I/O. Keep an opt-out copy mode for operators that need a
+        # detached file, but make the lock-protected direct path the default.
+        return direct_manifest(source) if direct_snapshot?
+
+        FileUtils.mkdir_p(@directory)
+        destination = File.join(@directory, "snapshot-#{Process.pid}-#{SecureRandom.hex(12)}.db")
+        FileUtils.copy_file(source, destination, true)
+        File.open(destination, "r+b") do |file|
+          file.flush
+          file.fsync
+        end
+
+        snapshot = manifest_for(
+          destination,
+          snapshot_id: File.basename(destination, ".db"),
+          file_sha256: Digest::SHA256.file(destination).hexdigest,
+          temporary: true
+        )
+        @lock.synchronize { @active_paths[destination] = true }
+        snapshot
+      # Remove a partially copied snapshot even when the caller is interrupted.
+      # rubocop:disable Lint/RescueException
+      rescue Exception
+        delete_file(destination) if destination
+        raise
+        # rubocop:enable Lint/RescueException
+      end
+
+      def direct_manifest(source)
+        manifest_for(
+          source,
+          snapshot_id: "live-#{Process.pid}-#{SecureRandom.hex(8)}",
+          file_sha256: nil,
+          temporary: false
+        )
+      end
+
+      def manifest_for(path, snapshot_id:, file_sha256:, temporary:)
+        {
+          format_version: FORMAT_VERSION,
+          snapshot_id: snapshot_id,
+          snapshot_path: File.expand_path(path),
+          page_size: @engine.storage_manager.page_size,
+          page_count: @engine.storage_manager.file_manager.num_pages,
+          file_sha256: file_sha256,
+          tables: table_manifest,
+          hidden_row_ids: @engine.accelerator_snapshot_hidden_row_ids,
+          temporary: temporary
+        }
+      end
+
+      def direct_snapshot?
+        accelerator_config = @engine.config[:accelerator] || @engine.config["accelerator"] || {}
+        configured = accelerator_config[:direct_snapshot] || accelerator_config["direct_snapshot"]
+        configured = ENV.fetch("RUBYDB_ACCELERATOR_DIRECT_SNAPSHOT", "on") if configured.nil?
+        !%w[off false 0].include?(configured.to_s.downcase)
+      end
+
+      def table_manifest
+        @engine.table_metadata.each_with_object({}) do |(table_name, metadata), tables|
+          columns = Array(metadata[:columns]).map do |column|
+            {
+              name: column.name.to_s,
+              type: column.type_class.to_s,
+              nullable: column.nullable?,
+              default: json_value(column.default)
+            }
+          end
+          indexes = Array(@engine.index_manager&.get_indexes_for_table(table_name)).filter_map do |index|
+            next unless index.type.to_sym == :btree && index.respond_to?(:snapshot_entries)
+
+            {
+              name: index.name.to_s,
+              type: index.type.to_s,
+              columns: index.columns.map(&:to_s),
+              unique: index.unique,
+              entries: index.snapshot_entries.map do |entry|
+                {key: json_value(entry[:key]), row_id: entry[:value].to_i}
+              end
+            }
+          end
+
+          tables[table_name.to_s] = {
+            pages: Array(@engine.table_pages_for_snapshot(table_name)).map(&:to_i),
+            columns: columns,
+            indexes: indexes,
+            row_count: metadata[:row_count].to_i
+          }
+        end
+      end
+
+      def json_value(value)
+        JSON.parse(JSON.generate(value))
+      rescue
+        nil
+      end
+
+      def release(snapshot)
+        path = snapshot && snapshot[:snapshot_path]
+        return unless path && snapshot[:temporary]
+
+        @lock.synchronize do
+          @active_paths.delete(path)
+          delete_file(path)
+          Dir.rmdir(@directory) if Dir.exist?(@directory) && Dir.empty?(@directory)
+        end
+      end
+
+      def delete_file(path)
+        File.delete(path) if path && File.file?(path)
+      rescue SystemCallError
+        nil
+      end
+    end
+  end
+end

data/lib/rubydb/version.rb

@@ -11,7 +11,7 @@ module RubyDB
   # - MAJOR: Incompatible API changes
   # - MINOR: Backwards-compatible new functionality
   # - PATCH: Backwards-compatible bug fixes
-  VERSION = "0.1.5"
+  VERSION = "0.1.6"
 
   # Version components for easy access
   module Version

data/lib/rubydb/wal/archive.rb

@@ -3,6 +3,7 @@
 require "fileutils"
 require "zlib"
 require "time"
+require_relative "../accelerator"
 
 module RubyDB
   module WAL
@@ -14,6 +15,7 @@ module RubyDB
         @wal_dir = wal_dir
         @archive_dir = archive_dir || File.join(wal_dir, "archive")
         @config = config
+        @accelerator = RubyDB::Accelerator::Client.new(config[:accelerator] || {})
         @compression = config[:compression] || true
         @max_archive_size = config[:max_size] || 10 * 1024 * 1024 * 1024  # 10GB
         @retention_days = config[:retention_days] || 30
@@ -170,9 +172,19 @@ module RubyDB
         end
       end
 
+      def close
+        @accelerator&.close
+        true
+      end
+
       private
 
       def compress_file(source, destination)
+        if (compressed = @accelerator.gzip(File.binread(source), level: @config[:compression_level]))
+          File.binwrite(destination, compressed)
+          return
+        end
+
         File.open(source, "rb") do |input|
           Zlib::GzipWriter.open(destination) do |output|
             output.write(input.read)
@@ -181,6 +193,11 @@ module RubyDB
       end
 
       def decompress_file(source, destination)
+        if (decompressed = @accelerator.gunzip(File.binread(source)))
+          File.binwrite(destination, decompressed)
+          return
+        end
+
         Zlib::GzipReader.open(source) do |input|
           File.binwrite(destination, input.read)
         end

data/lib/rubydb/wal/wal.rb

@@ -339,6 +339,7 @@ module RubyDB
           @checkpoint&.stop
           @writer.shutdown(wait)
           @reader&.close
+          @archive&.close
         end
       end
 

data/rubydb.gemspec

@@ -42,8 +42,18 @@ Gem::Specification.new do |spec|
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files = Dir.chdir(__dir__) do
-    `git ls-files -z`.split("\x0").reject do |f|
-      f.match(%r{^(test|spec|features|benchmarks|fuzz|chaos|examples|adapters/python)/})
+    tracked_files = `git ls-files -z`.split("\x0")
+    accelerator_files = Dir[
+      "accelerator/bin/rubydb-accelerator-*",
+      "accelerator/bin/SHA256SUMS",
+      "accelerator/cmd/**/*.go",
+      "accelerator/internal/**/*.go",
+      "accelerator/go.mod"
+    ]
+    runtime_files = Dir["lib/rubydb/accelerator.rb", "lib/rubydb/accelerator/**/*.rb", "lib/rubydb/cli/commands/accelerator.rb", "lib/rubydb/storage/snapshot_reader.rb"]
+    physical_execution_files = Dir["lib/rubydb/execution/physical_plan.rb", "lib/rubydb/execution/cost_model.rb", "lib/rubydb/execution/operator_selection.rb", "lib/rubydb/execution/accelerator_dispatch.rb"]
+    (tracked_files + accelerator_files + runtime_files + physical_execution_files).uniq.select { |f| File.file?(f) }.reject do |f|
+      f.match?(%r{^(test|spec|features|benchmarks|fuzz|chaos|examples|adapters/python)/}) || f.end_with?("_test.go")
     end
   end
   spec.bindir = "exe"