rubydb 0.1.5 → 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +4 -0
  3. data/CHANGELOG.md +14 -0
  4. data/Gemfile.lock +1 -1
  5. data/README.md +296 -227
  6. data/Rakefile +6 -1
  7. data/accelerator/bin/SHA256SUMS +6 -0
  8. data/accelerator/bin/rubydb-accelerator-darwin-amd64 +0 -0
  9. data/accelerator/bin/rubydb-accelerator-darwin-arm64 +0 -0
  10. data/accelerator/bin/rubydb-accelerator-linux-amd64 +0 -0
  11. data/accelerator/bin/rubydb-accelerator-linux-arm64 +0 -0
  12. data/accelerator/bin/rubydb-accelerator-windows-amd64.exe +0 -0
  13. data/accelerator/bin/rubydb-accelerator-windows-arm64.exe +0 -0
  14. data/accelerator/cmd/rubydb-accelerator/main.go +11 -0
  15. data/accelerator/go.mod +3 -0
  16. data/accelerator/internal/execution/aggregate.go +94 -0
  17. data/accelerator/internal/execution/distinct.go +22 -0
  18. data/accelerator/internal/execution/filter.go +73 -0
  19. data/accelerator/internal/execution/join.go +79 -0
  20. data/accelerator/internal/execution/operators.go +167 -0
  21. data/accelerator/internal/execution/scan.go +20 -0
  22. data/accelerator/internal/execution/sort.go +62 -0
  23. data/accelerator/internal/execution/types.go +136 -0
  24. data/accelerator/internal/execution/value.go +67 -0
  25. data/accelerator/internal/memory/arena.go +47 -0
  26. data/accelerator/internal/memory/reuse.go +22 -0
  27. data/accelerator/internal/metrics/registry.go +67 -0
  28. data/accelerator/internal/parallel/bounded_queue.go +56 -0
  29. data/accelerator/internal/parallel/scheduler.go +47 -0
  30. data/accelerator/internal/parallel/worker_pool.go +53 -0
  31. data/accelerator/internal/protocol/cancellation.go +48 -0
  32. data/accelerator/internal/protocol/columnar.go +263 -0
  33. data/accelerator/internal/protocol/frame.go +187 -0
  34. data/accelerator/internal/runtime/worker.go +521 -0
  35. data/accelerator/internal/storage/page_reader.go +81 -0
  36. data/accelerator/internal/storage/snapshot_scan.go +539 -0
  37. data/accelerator/internal/wal/checksum.go +13 -0
  38. data/accelerator/internal/wal/compression.go +41 -0
  39. data/accelerator/internal/wal/group_commit.go +24 -0
  40. data/accelerator/internal/wal/record_encoder.go +40 -0
  41. data/adapters/activerecord/README.md +8 -3
  42. data/adapters/activerecord/lib/active_record/connection_adapters/rubydb_adapter.rb +50 -42
  43. data/adapters/activerecord/rubydb-activerecord.gemspec +1 -1
  44. data/docs/README.md +3 -1
  45. data/docs/architecture/go-accelerator.md +179 -0
  46. data/docs/cli.md +24 -0
  47. data/docs/contributing/benchmarking.md +16 -0
  48. data/docs/developer/local-development.md +32 -0
  49. data/docs/release.md +2 -2
  50. data/lessons/02-local-development.md +2 -2
  51. data/lessons/04-rails-complex-apps.md +2 -2
  52. data/lessons/05-rubydb-production-server.md +2 -2
  53. data/lessons/07-hybrid-microservices.md +175 -90
  54. data/lessons/10-release-readiness.md +184 -117
  55. data/lessons/11-community-adapter.md +323 -0
  56. data/lessons/12-rails-ecommerce-pressure.md +263 -0
  57. data/lib/rubydb/accelerator/client.rb +451 -0
  58. data/lib/rubydb/accelerator/error.rb +22 -0
  59. data/lib/rubydb/accelerator/manager.rb +606 -0
  60. data/lib/rubydb/accelerator.rb +13 -0
  61. data/lib/rubydb/cli/application.rb +6 -1
  62. data/lib/rubydb/cli/commands/accelerator.rb +72 -0
  63. data/lib/rubydb/cli/commands/doctor.rb +3 -0
  64. data/lib/rubydb/client/client.rb +7 -0
  65. data/lib/rubydb/client/connection.rb +15 -0
  66. data/lib/rubydb/client/result.rb +5 -1
  67. data/lib/rubydb/configuration/defaults.rb +12 -0
  68. data/lib/rubydb/configuration/validation.rb +8 -1
  69. data/lib/rubydb/execution/accelerator_dispatch.rb +30 -0
  70. data/lib/rubydb/execution/cost_model.rb +72 -0
  71. data/lib/rubydb/execution/executor.rb +373 -11
  72. data/lib/rubydb/execution/operator_selection.rb +57 -0
  73. data/lib/rubydb/execution/physical_plan.rb +47 -0
  74. data/lib/rubydb/execution/planner.rb +12 -46
  75. data/lib/rubydb/execution/sort_executor.rb +22 -8
  76. data/lib/rubydb/indexes/btree.rb +31 -2
  77. data/lib/rubydb/rubydb.rb +7 -1
  78. data/lib/rubydb/server/session.rb +45 -0
  79. data/lib/rubydb/storage/engine.rb +74 -12
  80. data/lib/rubydb/storage/snapshot_reader.rb +167 -0
  81. data/lib/rubydb/version.rb +1 -1
  82. data/lib/rubydb/wal/archive.rb +17 -0
  83. data/lib/rubydb/wal/wal.rb +1 -0
  84. data/rubydb.gemspec +12 -2
  85. data/scripts/build_accelerator +49 -0
  86. data/scripts/release +34 -4
  87. data/scripts/replication_failover_drill +2 -2
  88. metadata +49 -1
@@ -0,0 +1,72 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RubyDB
4
+ module CLI
5
+ module Commands
6
+ # Accelerator - Inspect and verify the optional Go execution layer.
7
+ class Accelerator
8
+ def initialize(output, formatter)
9
+ @output = output
10
+ @formatter = formatter
11
+ end
12
+
13
+ def execute(args, options)
14
+ parser = OptionParser.new do |opts|
15
+ opts.banner = "Usage: rubydb accelerator [options]"
16
+ opts.on("--ping", "Start the accelerator and verify the protocol") do
17
+ options[:ping] = true
18
+ end
19
+ opts.on("--mode MODE", %w[auto off required], "Override mode for this check") do |mode|
20
+ options[:accelerator_mode] = mode
21
+ end
22
+ opts.on("--json", "Output JSON") do
23
+ options[:json] = true
24
+ end
25
+ opts.on("-h", "--help", "Show help") do
26
+ @output.puts opts
27
+ exit(0)
28
+ end
29
+ end
30
+ parser.parse!(args)
31
+
32
+ configured_accelerator = RubyDB::Configuration::Config.instance.to_hash[:accelerator]
33
+ config = if configured_accelerator.is_a?(Hash)
34
+ configured_accelerator.to_h.dup
35
+ else
36
+ # RUBYDB_ACCELERATOR=off|auto|required is a shorthand environment
37
+ # override. The environment parser may therefore replace the
38
+ # nested accelerator hash with a string; normalize it back into
39
+ # the client configuration instead of calling Hash#to_h on it.
40
+ {mode: configured_accelerator.to_s}
41
+ end
42
+ config[:mode] = options[:accelerator_mode] if options[:accelerator_mode]
43
+ client = RubyDB::Accelerator::Client.new(config)
44
+ details = client.stats
45
+ if options[:ping] && details[:available]
46
+ details[:ping] = client.ping
47
+ end
48
+ details[:healthy] = if details[:mode] == "off"
49
+ true
50
+ elsif details[:available]
51
+ !options[:ping] || details[:ping].is_a?(Hash)
52
+ else
53
+ details[:mode] != "required"
54
+ end
55
+
56
+ if options[:json]
57
+ @output.json(details)
58
+ else
59
+ @output.puts "RubyDB Go accelerator"
60
+ details.each { |key, value| @output.puts " #{key}: #{value.inspect}" }
61
+ end
62
+ details[:healthy] ? 0 : 1
63
+ rescue RubyDB::Accelerator::Error => error
64
+ @output.error(error.message)
65
+ 1
66
+ ensure
67
+ client&.close
68
+ end
69
+ end
70
+ end
71
+ end
72
+ end
@@ -42,6 +42,9 @@ module RubyDB
42
42
  checks << {name: "Connection", passed: true, details: {database: db_path, latency_ms: latency_ms}}
43
43
  checks << {name: "Storage", passed: engine.storage_manager.open?, details: engine.storage_manager.stats}
44
44
  checks << {name: "WAL", passed: !engine.wal.nil?, details: engine.wal.stats}
45
+ accelerator = engine.accelerator.stats
46
+ accelerator_passed = accelerator[:mode] != "required" || accelerator[:available]
47
+ checks << {name: "Go accelerator", passed: accelerator_passed, details: accelerator}
45
48
  unless options[:quick]
46
49
  checks << {name: "Indexes", passed: !engine.index_manager.nil?, details: engine.index_manager.stats}
47
50
  end
@@ -140,6 +140,13 @@ module RubyDB
140
140
  end
141
141
  end
142
142
 
143
+ def metadata(table_name = nil)
144
+ @lock.synchronize do
145
+ ensure_connected
146
+ @connection.send_metadata(table_name)
147
+ end
148
+ end
149
+
143
150
  def query_async(sql, params = [], timeout: nil)
144
151
  @lock.synchronize do
145
152
  ensure_connected
@@ -150,6 +150,21 @@ module RubyDB
150
150
  end
151
151
  end
152
152
 
153
+ # Fetch catalog metadata over the same authenticated connection used by
154
+ # SQL queries. Rails adapters need this for schema discovery in
155
+ # client/server mode; asking the server for it avoids pretending that a
156
+ # RubyDB catalog is SQLite's sqlite_master table.
157
+ def send_metadata(table_name = nil)
158
+ @lock.synchronize do
159
+ ensure_connected
160
+
161
+ payload = {}
162
+ payload[:table] = table_name.to_s if table_name
163
+ send_message(Protocol::Message.new(:metadata, payload))
164
+ unwrap_result(receive_message.payload)
165
+ end
166
+ end
167
+
153
168
  # Starts a query and returns a handle that can issue a wire-level
154
169
  # cancellation while the server is executing it. The same connection
155
170
  # must not be used for another application request until wait returns.
@@ -5,7 +5,7 @@ module RubyDB
5
5
  # Result - Query result set
6
6
  class Result
7
7
  attr_reader :columns, :rows, :row_count, :affected_rows, :sql
8
- attr_reader :command_tag, :statement_id, :transaction_id
8
+ attr_reader :command_tag, :statement_id, :transaction_id, :row_id, :inserted_id
9
9
  attr_reader :error
10
10
 
11
11
  def initialize(data = {})
@@ -17,6 +17,8 @@ module RubyDB
17
17
  @command_tag = data[:command_tag] || "SELECT"
18
18
  @statement_id = data[:statement_id]
19
19
  @transaction_id = data[:transaction_id]
20
+ @row_id = data[:row_id]
21
+ @inserted_id = data[:inserted_id]
20
22
  @success = data[:success] != false
21
23
  @error = data[:error]
22
24
  @warnings = data[:warnings] || []
@@ -85,6 +87,8 @@ module RubyDB
85
87
  row_count: @row_count,
86
88
  affected_rows: @affected_rows,
87
89
  command_tag: @command_tag,
90
+ row_id: @row_id,
91
+ inserted_id: @inserted_id,
88
92
  success: @success,
89
93
  error: @error,
90
94
  execution_time_ms: @execution_time_ms
@@ -119,6 +119,18 @@ module RubyDB
119
119
  debug: false,
120
120
  profile: false,
121
121
  fake_data: false
122
+ },
123
+
124
+ # Bundled Go acceleration. Ruby remains the correctness path and
125
+ # automatically handles operations the accelerator cannot prove safe.
126
+ accelerator: {
127
+ mode: "auto",
128
+ timeout: 30,
129
+ max_frame_size: 16 * 1024 * 1024,
130
+ min_rows: 256,
131
+ read_pipeline: "on",
132
+ direct_snapshot: true,
133
+ binary: nil
122
134
  }
123
135
  }
124
136
 
@@ -77,7 +77,14 @@ module RubyDB
77
77
  "monitoring.health_check_interval" => {type: :integer, required: true, min: 1},
78
78
  "monitoring.stats_collector" => {type: :boolean, required: true},
79
79
  "monitoring.prometheus" => {type: :boolean, required: true},
80
- "monitoring.prometheus_port" => {type: :integer, required: true, min: 1, max: 65535}
80
+ "monitoring.prometheus_port" => {type: :integer, required: true, min: 1, max: 65535},
81
+
82
+ "accelerator.mode" => {type: :string, required: true, enum: ["auto", "off", "required"]},
83
+ "accelerator.timeout" => {type: :integer, required: true, min: 1},
84
+ "accelerator.max_frame_size" => {type: :integer, required: true, min: 1024},
85
+ "accelerator.min_rows" => {type: :integer, required: true, min: 0},
86
+ "accelerator.read_pipeline" => {type: :string, required: true, enum: ["off", "on"]},
87
+ "accelerator.direct_snapshot" => {type: :boolean, required: true}
81
88
  }
82
89
 
83
90
  def initialize
@@ -0,0 +1,30 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RubyDB
4
+ module Execution
5
+ # Keeps capability checks and result validation outside the executor's SQL
6
+ # logic. The Go path is an optimization; this object never makes it the
7
+ # source of truth.
8
+ class AcceleratorDispatch
9
+ def initialize(accelerator)
10
+ @accelerator = accelerator
11
+ end
12
+
13
+ def preferred?(workload)
14
+ @accelerator&.manager&.acceleration_preferred?(workload)
15
+ end
16
+
17
+ def equivalent?(left, right)
18
+ normalize(left) == normalize(right)
19
+ end
20
+
21
+ private
22
+
23
+ def normalize(rows)
24
+ rows.map do |row|
25
+ row.each_with_object({}) { |(key, value), normalized| normalized[key.to_s] = value }
26
+ end
27
+ end
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,72 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RubyDB
4
+ module Execution
5
+ # Conservative planner estimates. They are deliberately independent from
6
+ # the Go runtime: a wrong estimate must never change SQL semantics.
7
+ class CostModel
8
+ attr_reader :engine
9
+
10
+ def initialize(engine)
11
+ @engine = engine
12
+ end
13
+
14
+ def apply(plan)
15
+ row_count = table_row_count(plan.table_name)
16
+ selectivity = estimate_selectivity(plan.predicate)
17
+ estimated_rows = [(row_count * selectivity).ceil, 0].max
18
+ cost = case plan.scan_type
19
+ when :index
20
+ plan.predicate ? estimated_rows * 0.1 : row_count * 0.5
21
+ when :sequential
22
+ row_count
23
+ else
24
+ row_count
25
+ end
26
+ cost += sort_cost(estimated_rows) if plan.order_by&.any?
27
+ cost += estimated_rows if plan.aggregates&.any?
28
+ plan.set_cost(cost, estimated_rows)
29
+ end
30
+
31
+ def estimate_selectivity(predicate)
32
+ return 1.0 unless predicate
33
+
34
+ case predicate
35
+ when Predicate::Comparison
36
+ case predicate.operator
37
+ when :EQ then 0.01
38
+ when :NE then 0.9
39
+ when :LT, :LTE, :GT, :GTE then 0.5
40
+ else 0.5
41
+ end
42
+ when Predicate::And
43
+ estimate_selectivity(predicate.left) * estimate_selectivity(predicate.right)
44
+ when Predicate::Or
45
+ left = estimate_selectivity(predicate.left)
46
+ right = estimate_selectivity(predicate.right)
47
+ left + right - (left * right)
48
+ when Predicate::Not
49
+ 1.0 - estimate_selectivity(predicate.operand)
50
+ else
51
+ 0.5
52
+ end
53
+ end
54
+
55
+ private
56
+
57
+ def table_row_count(table_name)
58
+ return 0 unless table_name
59
+
60
+ Integer(@engine.table_row_count(table_name))
61
+ rescue
62
+ 0
63
+ end
64
+
65
+ def sort_cost(rows)
66
+ return 0 if rows < 2
67
+
68
+ rows * Math.log2(rows)
69
+ end
70
+ end
71
+ end
72
+ end