rubydb 0.1.4 → 0.1.6

data/ROADMAP.md

@@ -1,27 +1,27 @@
-# RubyDB roadmap
-
-## Current phase: validated foundation
-
-RubyDB has a durable storage/WAL/recovery foundation, transactions and MVCC,
-the documented SQL engine, server/client protocol, Rails adapter coverage,
-backups, logical replication, monitoring, fuzzing, and release automation.
-The latest local audit passed 270 examples with zero failures.
-
-## Next gates
-
-1. Retain hosted Ruby/Rails/OS matrix evidence for each release.
-2. Run physical filesystem quota, power-loss, corruption, restore, and compaction
-   drills on deployment targets.
-3. Validate multi-host replication partitions, independent fencing, stale
-   primary rejection, and operator promotion procedures.
-4. Complete certificate/secret rotation validation and an independent security
-   review.
-5. Establish workload-specific latency, throughput, capacity, RPO, and RTO
-   baselines.
-
-## Longer term
-
-Expand the documented SQLite-style profile and only then consider broader
-PostgreSQL/MySQL compatibility where implementation and semantic validation can
-be maintained. Add automatic election only with an independently tested fencing
-authority. Preserve correctness and explicit unsupported errors as hard gates.
+# RubyDB roadmap
+
+## Current phase: validated foundation
+
+RubyDB has a durable storage/WAL/recovery foundation, transactions and MVCC,
+the documented SQL engine, server/client protocol, Rails adapter coverage,
+backups, logical replication, monitoring, fuzzing, and release automation.
+The latest local audit passed 270 examples with zero failures.
+
+## Next gates
+
+1. Retain hosted Ruby/Rails/OS matrix evidence for each release.
+2. Run physical filesystem quota, power-loss, corruption, restore, and compaction
+   drills on deployment targets.
+3. Validate multi-host replication partitions, independent fencing, stale
+   primary rejection, and operator promotion procedures.
+4. Complete certificate/secret rotation validation and an independent security
+   review.
+5. Establish workload-specific latency, throughput, capacity, RPO, and RTO
+   baselines.
+
+## Longer term
+
+Expand the documented SQLite-style profile and only then consider broader
+PostgreSQL/MySQL compatibility where implementation and semantic validation can
+be maintained. Add automatic election only with an independently tested fencing
+authority. Preserve correctness and explicit unsupported errors as hard gates.

data/Rakefile

@@ -1,71 +1,76 @@
-# frozen_string_literal: true
-
-require 'bundler/gem_tasks'
-require 'rspec/core/rake_task'
-require 'rbconfig'
-require 'digest'
-require 'fileutils'
-
-RSpec::Core::RakeTask.new(:spec)
-
-task default: %i[spec rubocop]
-
-namespace :build do
-  desc 'Build the gem and write a SHA-512 checksum for the exact artifact'
-  task checksum: :build do
-    gem_path = Dir['pkg/rubydb-*.gem'].max_by { |path| File.mtime(path) }
-    abort 'No built gem found in pkg/' unless gem_path
-
-    FileUtils.mkdir_p('checksums')
-    checksum_path = File.join('checksums', "#{File.basename(gem_path)}.sha512")
-    File.write(checksum_path, "#{Digest::SHA512.file(gem_path).hexdigest}  #{File.basename(gem_path)}\n")
-    puts "Wrote #{checksum_path}"
-  end
-end
-
-desc 'Run all tests'
-task test: :spec
-
-desc 'Run unit tests only'
-RSpec::Core::RakeTask.new(:'test:unit') do |t|
-  t.pattern = 'test/unit/**/*_spec.rb'
-end
-
-desc 'Run integration tests only'
-RSpec::Core::RakeTask.new(:'test:integration') do |t|
-  t.pattern = 'test/integration/**/*_spec.rb'
-end
-
-begin
-  require 'rubocop/rake_task'
-  RuboCop::RakeTask.new
-rescue LoadError
-  # rubocop not installed
-end
-
-namespace :test do
-  desc 'Run concurrency tests'
-  RSpec::Core::RakeTask.new(:concurrency) do |t|
-    t.pattern = 'test/concurrency/**/*_spec.rb'
-  end
-
-  desc 'Run crash tests'
-  RSpec::Core::RakeTask.new(:crash) do |t|
-    t.pattern = 'test/crash/**/*_spec.rb'
-  end
-
-  desc 'Run chaos tests'
-  RSpec::Core::RakeTask.new(:chaos) do |t|
-    t.pattern = 'test/chaos/**/*_spec.rb'
-  end
-end
-
-desc 'Run benchmarks'
-task :benchmark do
-  Dir['benchmarks/**/*.rb'].each { |f| load f }
-end
-
-desc 'Run the concurrent workload and durability verification'
-task :workload do
-  abort 'Concurrent workload failed' unless system(RbConfig.ruby, 'benchmarks/concurrent_workload.rb')
-end
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+require "rbconfig"
+require "digest"
+require "fileutils"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: %i[spec rubocop]
+
+namespace :build do
+  desc "Build the bundled Go accelerator for the current platform"
+  task :accelerator do
+    abort "Go accelerator build failed" unless system({"RUBYDB_ACCELERATOR_TARGETS" => "current"}, RbConfig.ruby, "scripts/build_accelerator")
+  end
+
+  desc "Build the gem and write a SHA-512 checksum for the exact artifact"
+  task checksum: [:accelerator, :build] do
+    gem_path = Dir["pkg/rubydb-*.gem"].max_by { |path| File.mtime(path) }
+    abort "No built gem found in pkg/" unless gem_path
+
+    FileUtils.mkdir_p("checksums")
+    checksum_path = File.join("checksums", "#{File.basename(gem_path)}.sha512")
+    File.write(checksum_path, "#{Digest::SHA512.file(gem_path).hexdigest}  #{File.basename(gem_path)}\n")
+    puts "Wrote #{checksum_path}"
+  end
+end
+
+desc "Run all tests"
+task test: :spec
+
+desc "Run unit tests only"
+RSpec::Core::RakeTask.new(:"test:unit") do |t|
+  t.pattern = "test/unit/**/*_spec.rb"
+end
+
+desc "Run integration tests only"
+RSpec::Core::RakeTask.new(:"test:integration") do |t|
+  t.pattern = "test/integration/**/*_spec.rb"
+end
+
+begin
+  require "rubocop/rake_task"
+  RuboCop::RakeTask.new
+rescue LoadError
+  # rubocop not installed
+end
+
+namespace :test do
+  desc "Run concurrency tests"
+  RSpec::Core::RakeTask.new(:concurrency) do |t|
+    t.pattern = "test/concurrency/**/*_spec.rb"
+  end
+
+  desc "Run crash tests"
+  RSpec::Core::RakeTask.new(:crash) do |t|
+    t.pattern = "test/crash/**/*_spec.rb"
+  end
+
+  desc "Run chaos tests"
+  RSpec::Core::RakeTask.new(:chaos) do |t|
+    t.pattern = "test/chaos/**/*_spec.rb"
+  end
+end
+
+desc "Run benchmarks"
+task :benchmark do
+  Dir["benchmarks/**/*.rb"].each { |f| load f }
+end
+
+desc "Run the concurrent workload and durability verification"
+task :workload do
+  abort "Concurrent workload failed" unless system(RbConfig.ruby, "benchmarks/concurrent_workload.rb")
+end

data/SECURITY.md

@@ -1,54 +1,54 @@
-# RubyDB security policy
-
-RubyDB is alpha software with a documented, tested feature set. Security
-controls are implemented and tested for the server protocol, TLS transport,
-password/SCRAM authentication, authorization, bounded frames, query deadlines,
-resource limits, replication tokens, path validation, backup manifests, and
-fencing. A deployment must still complete an independent security review
-before it handles sensitive or regulated data.
-
-## Secure deployment requirements
-
-- Keep the database directory, WAL, backups, TLS private keys, authentication
-  credentials, and replication state on protected storage with least-privilege
-  filesystem ownership.
-- Bind the server and replication listeners to explicit private addresses. Do
-  not expose them directly to the public internet; restrict ingress with a
-  firewall or private network.
-- Enable TLS with TLS 1.2 or newer, validate the server certificate, and use a
-  trusted CA or mutual TLS where the deployment requires peer identity.
-- Use high-entropy credentials and a separate high-entropy replication token.
-  Never put secrets in source control, URLs, shell history, logs, or bug
-  reports.
-- Configure maximum connections, request/frame sizes, query deadlines, and
-  filesystem thresholds for the host. Treat readiness failure and recovery-
-  required acknowledgements as incidents.
-- Run backups with encryption approved by the organization, retain the
-  manifest and WAL chain together, and perform restore drills into a new
-  inactive directory.
-
-## Rotation and incident response
-
-Stage replacement certificates and credentials, validate them on a non-serving
-instance, then rotate during a planned window. Drain clients or replication
-peers, change both sides of a replication token, reconnect, and verify the
-authentication-failure metric returns to baseline. Retain the old certificate
-only for the documented overlap window and remove it afterward.
-
-For suspected compromise, corruption, checksum failure, replication
-divergence, or fencing anomalies: stop writes, isolate the affected node,
-preserve the database/WAL/log files and timestamps, and create a verified copy
-before attempting repair, vacuum, restore, or promotion. Automatic election is
-disabled; never run two writable primaries without an externally verified
-fencing lease.
-
-## Reporting a vulnerability
-
-Do not disclose unpatched vulnerabilities in a public issue. Use a private
-GitHub security advisory for the repository or contact the maintainer listed in
-the gem metadata with reproduction steps, affected versions, impact, and a
-safe disclosure timeline. Do not include production secrets or customer data.
-
-Security scans run in GitHub Actions, but scan results do not replace review of
-deployment configuration, network policy, secret storage, certificate
-lifecycle, or the host filesystem.
+# RubyDB security policy
+
+RubyDB is alpha software with a documented, tested feature set. Security
+controls are implemented and tested for the server protocol, TLS transport,
+password/SCRAM authentication, authorization, bounded frames, query deadlines,
+resource limits, replication tokens, path validation, backup manifests, and
+fencing. A deployment must still complete an independent security review
+before it handles sensitive or regulated data.
+
+## Secure deployment requirements
+
+- Keep the database directory, WAL, backups, TLS private keys, authentication
+  credentials, and replication state on protected storage with least-privilege
+  filesystem ownership.
+- Bind the server and replication listeners to explicit private addresses. Do
+  not expose them directly to the public internet; restrict ingress with a
+  firewall or private network.
+- Enable TLS with TLS 1.2 or newer, validate the server certificate, and use a
+  trusted CA or mutual TLS where the deployment requires peer identity.
+- Use high-entropy credentials and a separate high-entropy replication token.
+  Never put secrets in source control, URLs, shell history, logs, or bug
+  reports.
+- Configure maximum connections, request/frame sizes, query deadlines, and
+  filesystem thresholds for the host. Treat readiness failure and recovery-
+  required acknowledgements as incidents.
+- Run backups with encryption approved by the organization, retain the
+  manifest and WAL chain together, and perform restore drills into a new
+  inactive directory.
+
+## Rotation and incident response
+
+Stage replacement certificates and credentials, validate them on a non-serving
+instance, then rotate during a planned window. Drain clients or replication
+peers, change both sides of a replication token, reconnect, and verify the
+authentication-failure metric returns to baseline. Retain the old certificate
+only for the documented overlap window and remove it afterward.
+
+For suspected compromise, corruption, checksum failure, replication
+divergence, or fencing anomalies: stop writes, isolate the affected node,
+preserve the database/WAL/log files and timestamps, and create a verified copy
+before attempting repair, vacuum, restore, or promotion. Automatic election is
+disabled; never run two writable primaries without an externally verified
+fencing lease.
+
+## Reporting a vulnerability
+
+Do not disclose unpatched vulnerabilities in a public issue. Use a private
+GitHub security advisory for the repository or contact the maintainer listed in
+the gem metadata with reproduction steps, affected versions, impact, and a
+safe disclosure timeline. Do not include production secrets or customer data.
+
+Security scans run in GitHub Actions, but scan results do not replace review of
+deployment configuration, network policy, secret storage, certificate
+lifecycle, or the host filesystem.

data/SUPPORT.md

@@ -1,14 +1,14 @@
-# RubyDB support policy
-
-RubyDB currently supports the documented RubyDB SQL surface, the tested common
-SQLite-style profile, the Ruby client/server path, and the ActiveRecord adapter
-combinations listed by CI. It does not promise complete PostgreSQL, MySQL, or
-SQLite dialect, file-format, extension, or wire compatibility.
-
-When requesting help, include RubyDB commit/version, Ruby and Rails versions,
-OS, topology, configuration shape without secrets, a minimal reproduction, and
-the exact error. Include workload and recovery evidence for data-safety issues.
-
-For corruption, checksum, replication divergence, or fencing anomalies, stop
-writes and preserve the database, WAL, logs, and timestamps before attempting
-repair. Report unpatched security issues privately through `SECURITY.md`.
+# RubyDB support policy
+
+RubyDB currently supports the documented RubyDB SQL surface, the tested common
+SQLite-style profile, the Ruby client/server path, and the ActiveRecord adapter
+combinations listed by CI. It does not promise complete PostgreSQL, MySQL, or
+SQLite dialect, file-format, extension, or wire compatibility.
+
+When requesting help, include RubyDB commit/version, Ruby and Rails versions,
+OS, topology, configuration shape without secrets, a minimal reproduction, and
+the exact error. Include workload and recovery evidence for data-safety issues.
+
+For corruption, checksum, replication divergence, or fencing anomalies, stop
+writes and preserve the database, WAL, logs, and timestamps before attempting
+repair. Report unpatched security issues privately through `SECURITY.md`.

data/accelerator/bin/SHA256SUMS

@@ -0,0 +1,6 @@
+ceeaad8692cfdc0e7b72b85d2c17b5e23c754d5f1159590df142fcfa857b8ff3  rubydb-accelerator-darwin-amd64
+1ab2ca6b7294f7d97ff4e816a5ad504802e84e4c196146a199a145a74829e3a0  rubydb-accelerator-darwin-arm64
+a0d141e3c41f2709d5bbaa67dd856f28805e4c2cbbc1034826e190877aa05aad  rubydb-accelerator-linux-amd64
+445b164c71284ce6432215fcf1213a99497194b3c8da6ea9577602405de4cf38  rubydb-accelerator-linux-arm64
+9e5c4f9acc9fe9751d5c945956fdbec50f9c698a2b5a0366255edef700a1499d  rubydb-accelerator-windows-amd64.exe
+3dd6aa97761f00a4552fa21a16571598bdfef8850347202ee4131d88ee97d002  rubydb-accelerator-windows-arm64.exe

data/accelerator/cmd/rubydb-accelerator/main.go

@@ -0,0 +1,11 @@
+package main
+
+import (
+	"os"
+
+	acceleratorRuntime "github.com/aldanedev-create/rubydb/accelerator/internal/runtime"
+)
+
+func main() {
+	acceleratorRuntime.Run(os.Stdin, os.Stdout)
+}

data/accelerator/go.mod

@@ -0,0 +1,3 @@
+module github.com/aldanedev-create/rubydb/accelerator
+
+go 1.23

data/accelerator/internal/execution/aggregate.go

@@ -0,0 +1,94 @@
+package execution
+
+import (
+	"encoding/json"
+	"strings"
+)
+
+func AggregateRows(rows []map[string]interface{}, groupBy []string, definitions []Aggregate) []map[string]interface{} {
+	groups := map[string][]map[string]interface{}{}
+	keys := map[string][]interface{}{}
+	order := make([]string, 0)
+	if len(groupBy) == 0 {
+		groups["_"] = rows
+		order = append(order, "_")
+	} else {
+		for _, row := range rows {
+			values := make([]interface{}, len(groupBy))
+			for i, column := range groupBy {
+				values[i] = row[column]
+			}
+			keyBytes, _ := json.Marshal(values)
+			key := string(keyBytes)
+			if _, exists := groups[key]; !exists {
+				order = append(order, key)
+			}
+			groups[key] = append(groups[key], row)
+			keys[key] = values
+		}
+	}
+	result := make([]map[string]interface{}, 0, len(groups))
+	for _, key := range order {
+		group := groups[key]
+		row := map[string]interface{}{}
+		for i, column := range groupBy {
+			row[column] = keys[key][i]
+		}
+		for _, definition := range definitions {
+			name := definition.Alias
+			if name == "" {
+				name = strings.ToLower(definition.Function) + "(" + definition.Column + ")"
+			}
+			row[name] = AggregateValue(group, definition)
+		}
+		result = append(result, row)
+	}
+	return result
+}
+
+func AggregateValue(rows []map[string]interface{}, definition Aggregate) interface{} {
+	values := make([]interface{}, 0, len(rows))
+	for _, row := range rows {
+		value := row[definition.Column]
+		if value != nil {
+			values = append(values, value)
+		}
+	}
+	switch strings.ToUpper(definition.Function) {
+	case "COUNT":
+		if definition.Column == "*" {
+			return len(rows)
+		}
+		return len(values)
+	case "SUM", "AVG":
+		var sum float64
+		for _, value := range values {
+			number, ok := NumberValue(value)
+			if !ok {
+				return nil
+			}
+			sum += number
+		}
+		if strings.EqualFold(definition.Function, "AVG") {
+			if len(values) == 0 {
+				return nil
+			}
+			return sum / float64(len(values))
+		}
+		return sum
+	case "MIN", "MAX":
+		if len(values) == 0 {
+			return nil
+		}
+		best := values[0]
+		for _, value := range values[1:] {
+			comparison := Compare(value, best)
+			if (strings.EqualFold(definition.Function, "MIN") && comparison < 0) || (strings.EqualFold(definition.Function, "MAX") && comparison > 0) {
+				best = value
+			}
+		}
+		return best
+	default:
+		return nil
+	}
+}

data/accelerator/internal/execution/distinct.go

@@ -0,0 +1,22 @@
+package execution
+
+import "encoding/json"
+
+func DistinctRows(rows []map[string]interface{}, columns []string) []map[string]interface{} {
+	seen := make(map[string]struct{}, len(rows))
+	result := make([]map[string]interface{}, 0, len(rows))
+	for _, row := range rows {
+		values := make([]interface{}, len(columns))
+		for index, column := range columns {
+			values[index] = row[column]
+		}
+		keyBytes, _ := json.Marshal(values)
+		key := string(keyBytes)
+		if _, exists := seen[key]; exists {
+			continue
+		}
+		seen[key] = struct{}{}
+		result = append(result, row)
+	}
+	return result
+}

data/accelerator/internal/execution/filter.go

@@ -0,0 +1,73 @@
+package execution
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+
+	"github.com/aldanedev-create/rubydb/accelerator/internal/parallel"
+)
+
+func FilterRows(rows []map[string]interface{}, condition Filter) []map[string]interface{} {
+	if len(rows) < 4096 {
+		filtered := make([]map[string]interface{}, 0, len(rows))
+		for _, row := range rows {
+			if Matches(row[condition.Column], condition.Operator, condition.Value) {
+				filtered = append(filtered, row)
+			}
+		}
+		return filtered
+	}
+
+	chunks := make([][]map[string]interface{}, parallel.ChunkCount(len(rows)))
+	parallel.ForEachChunk(len(rows), func(index, start, end int) {
+		filtered := make([]map[string]interface{}, 0, end-start)
+		for _, row := range rows[start:end] {
+			if Matches(row[condition.Column], condition.Operator, condition.Value) {
+				filtered = append(filtered, row)
+			}
+		}
+		chunks[index] = filtered
+	})
+	filtered := make([]map[string]interface{}, 0, len(rows))
+	for _, chunk := range chunks {
+		filtered = append(filtered, chunk...)
+	}
+	return filtered
+}
+
+func Matches(actual interface{}, operator string, expected interface{}) bool {
+	if strings.EqualFold(operator, "is_null") {
+		return actual == nil
+	}
+	if strings.EqualFold(operator, "is_not_null") {
+		return actual != nil
+	}
+	if actual == nil || expected == nil {
+		return false
+	}
+	if strings.EqualFold(operator, "like") {
+		pattern := regexp.QuoteMeta(fmt.Sprint(expected))
+		pattern = strings.ReplaceAll(pattern, "%", ".*")
+		pattern = strings.ReplaceAll(pattern, "_", ".")
+		matched, _ := regexp.MatchString("(?i)^"+pattern+"$", fmt.Sprint(actual))
+		return matched
+	}
+	comparison := Compare(actual, expected)
+	switch strings.ToLower(operator) {
+	case "eq":
+		return comparison == 0
+	case "ne":
+		return comparison != 0
+	case "lt":
+		return comparison < 0
+	case "lte":
+		return comparison <= 0
+	case "gt":
+		return comparison > 0
+	case "gte":
+		return comparison >= 0
+	default:
+		return false
+	}
+}

data/accelerator/internal/execution/join.go

@@ -0,0 +1,79 @@
+package execution
+
+import (
+	"encoding/json"
+	"sort"
+)
+
+func HashJoinRows(leftRows, rightRows []map[string]interface{}, leftKey, rightKey string) []map[string]interface{} {
+	index := make(map[string][]map[string]interface{}, len(rightRows))
+	for _, row := range rightRows {
+		key := JoinKey(row[rightKey])
+		index[key] = append(index[key], row)
+	}
+	rows := make([]map[string]interface{}, 0)
+	for _, left := range leftRows {
+		for _, right := range index[JoinKey(left[leftKey])] {
+			merged := make(map[string]interface{}, len(left)+len(right))
+			for key, value := range right {
+				merged[key] = value
+			}
+			for key, value := range left {
+				merged[key] = value
+			}
+			rows = append(rows, merged)
+		}
+	}
+	return rows
+}
+
+// MergeJoinRows is used when both inputs are already ordered by their join
+// keys. It avoids allocating a hash table for large, sorted relations.
+func MergeJoinRows(leftRows, rightRows []map[string]interface{}, leftKey, rightKey string) []map[string]interface{} {
+	left := append([]map[string]interface{}(nil), leftRows...)
+	right := append([]map[string]interface{}(nil), rightRows...)
+	sort.SliceStable(left, func(i, j int) bool { return Compare(left[i][leftKey], left[j][leftKey]) < 0 })
+	sort.SliceStable(right, func(i, j int) bool { return Compare(right[i][rightKey], right[j][rightKey]) < 0 })
+	result := make([]map[string]interface{}, 0)
+	for i, j := 0, 0; i < len(left) && j < len(right); {
+		comparison := Compare(left[i][leftKey], right[j][rightKey])
+		if comparison < 0 {
+			i++
+			continue
+		}
+		if comparison > 0 {
+			j++
+			continue
+		}
+		leftEnd, rightEnd := i, j
+		for leftEnd < len(left) && Compare(left[leftEnd][leftKey], left[i][leftKey]) == 0 {
+			leftEnd++
+		}
+		for rightEnd < len(right) && Compare(right[rightEnd][rightKey], right[j][rightKey]) == 0 {
+			rightEnd++
+		}
+		for leftIndex := i; leftIndex < leftEnd; leftIndex++ {
+			for rightIndex := j; rightIndex < rightEnd; rightIndex++ {
+				result = append(result, mergeRows(left[leftIndex], right[rightIndex]))
+			}
+		}
+		i, j = leftEnd, rightEnd
+	}
+	return result
+}
+
+func mergeRows(left, right map[string]interface{}) map[string]interface{} {
+	merged := make(map[string]interface{}, len(left)+len(right))
+	for key, value := range right {
+		merged[key] = value
+	}
+	for key, value := range left {
+		merged[key] = value
+	}
+	return merged
+}
+
+func JoinKey(value interface{}) string {
+	encoded, _ := json.Marshal(value)
+	return string(encoded)
+}