rubydb 0.1.5 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +4 -0
- data/CHANGELOG.md +14 -0
- data/Gemfile.lock +1 -1
- data/README.md +296 -227
- data/Rakefile +6 -1
- data/accelerator/bin/SHA256SUMS +6 -0
- data/accelerator/bin/rubydb-accelerator-darwin-amd64 +0 -0
- data/accelerator/bin/rubydb-accelerator-darwin-arm64 +0 -0
- data/accelerator/bin/rubydb-accelerator-linux-amd64 +0 -0
- data/accelerator/bin/rubydb-accelerator-linux-arm64 +0 -0
- data/accelerator/bin/rubydb-accelerator-windows-amd64.exe +0 -0
- data/accelerator/bin/rubydb-accelerator-windows-arm64.exe +0 -0
- data/accelerator/cmd/rubydb-accelerator/main.go +11 -0
- data/accelerator/go.mod +3 -0
- data/accelerator/internal/execution/aggregate.go +94 -0
- data/accelerator/internal/execution/distinct.go +22 -0
- data/accelerator/internal/execution/filter.go +73 -0
- data/accelerator/internal/execution/join.go +79 -0
- data/accelerator/internal/execution/operators.go +167 -0
- data/accelerator/internal/execution/scan.go +20 -0
- data/accelerator/internal/execution/sort.go +62 -0
- data/accelerator/internal/execution/types.go +136 -0
- data/accelerator/internal/execution/value.go +67 -0
- data/accelerator/internal/memory/arena.go +47 -0
- data/accelerator/internal/memory/reuse.go +22 -0
- data/accelerator/internal/metrics/registry.go +67 -0
- data/accelerator/internal/parallel/bounded_queue.go +56 -0
- data/accelerator/internal/parallel/scheduler.go +47 -0
- data/accelerator/internal/parallel/worker_pool.go +53 -0
- data/accelerator/internal/protocol/cancellation.go +48 -0
- data/accelerator/internal/protocol/columnar.go +263 -0
- data/accelerator/internal/protocol/frame.go +187 -0
- data/accelerator/internal/runtime/worker.go +521 -0
- data/accelerator/internal/storage/page_reader.go +81 -0
- data/accelerator/internal/storage/snapshot_scan.go +539 -0
- data/accelerator/internal/wal/checksum.go +13 -0
- data/accelerator/internal/wal/compression.go +41 -0
- data/accelerator/internal/wal/group_commit.go +24 -0
- data/accelerator/internal/wal/record_encoder.go +40 -0
- data/adapters/activerecord/README.md +8 -3
- data/adapters/activerecord/lib/active_record/connection_adapters/rubydb_adapter.rb +50 -42
- data/adapters/activerecord/rubydb-activerecord.gemspec +1 -1
- data/docs/README.md +3 -1
- data/docs/architecture/go-accelerator.md +179 -0
- data/docs/cli.md +24 -0
- data/docs/contributing/benchmarking.md +16 -0
- data/docs/developer/local-development.md +32 -0
- data/docs/release.md +2 -2
- data/lessons/02-local-development.md +2 -2
- data/lessons/04-rails-complex-apps.md +2 -2
- data/lessons/05-rubydb-production-server.md +2 -2
- data/lessons/07-hybrid-microservices.md +175 -90
- data/lessons/10-release-readiness.md +184 -117
- data/lessons/11-community-adapter.md +323 -0
- data/lessons/12-rails-ecommerce-pressure.md +263 -0
- data/lib/rubydb/accelerator/client.rb +451 -0
- data/lib/rubydb/accelerator/error.rb +22 -0
- data/lib/rubydb/accelerator/manager.rb +606 -0
- data/lib/rubydb/accelerator.rb +13 -0
- data/lib/rubydb/cli/application.rb +6 -1
- data/lib/rubydb/cli/commands/accelerator.rb +72 -0
- data/lib/rubydb/cli/commands/doctor.rb +3 -0
- data/lib/rubydb/client/client.rb +7 -0
- data/lib/rubydb/client/connection.rb +15 -0
- data/lib/rubydb/client/result.rb +5 -1
- data/lib/rubydb/configuration/defaults.rb +12 -0
- data/lib/rubydb/configuration/validation.rb +8 -1
- data/lib/rubydb/execution/accelerator_dispatch.rb +30 -0
- data/lib/rubydb/execution/cost_model.rb +72 -0
- data/lib/rubydb/execution/executor.rb +373 -11
- data/lib/rubydb/execution/operator_selection.rb +57 -0
- data/lib/rubydb/execution/physical_plan.rb +47 -0
- data/lib/rubydb/execution/planner.rb +12 -46
- data/lib/rubydb/execution/sort_executor.rb +22 -8
- data/lib/rubydb/indexes/btree.rb +31 -2
- data/lib/rubydb/rubydb.rb +7 -1
- data/lib/rubydb/server/session.rb +45 -0
- data/lib/rubydb/storage/engine.rb +74 -12
- data/lib/rubydb/storage/snapshot_reader.rb +167 -0
- data/lib/rubydb/version.rb +1 -1
- data/lib/rubydb/wal/archive.rb +17 -0
- data/lib/rubydb/wal/wal.rb +1 -0
- data/rubydb.gemspec +12 -2
- data/scripts/build_accelerator +49 -0
- data/scripts/release +34 -4
- data/scripts/replication_failover_drill +2 -2
- metadata +49 -1
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RubyDB
|
|
4
|
+
module CLI
|
|
5
|
+
module Commands
|
|
6
|
+
# Accelerator - Inspect and verify the optional Go execution layer.
|
|
7
|
+
class Accelerator
|
|
8
|
+
def initialize(output, formatter)
|
|
9
|
+
@output = output
|
|
10
|
+
@formatter = formatter
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def execute(args, options)
|
|
14
|
+
parser = OptionParser.new do |opts|
|
|
15
|
+
opts.banner = "Usage: rubydb accelerator [options]"
|
|
16
|
+
opts.on("--ping", "Start the accelerator and verify the protocol") do
|
|
17
|
+
options[:ping] = true
|
|
18
|
+
end
|
|
19
|
+
opts.on("--mode MODE", %w[auto off required], "Override mode for this check") do |mode|
|
|
20
|
+
options[:accelerator_mode] = mode
|
|
21
|
+
end
|
|
22
|
+
opts.on("--json", "Output JSON") do
|
|
23
|
+
options[:json] = true
|
|
24
|
+
end
|
|
25
|
+
opts.on("-h", "--help", "Show help") do
|
|
26
|
+
@output.puts opts
|
|
27
|
+
exit(0)
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
parser.parse!(args)
|
|
31
|
+
|
|
32
|
+
configured_accelerator = RubyDB::Configuration::Config.instance.to_hash[:accelerator]
|
|
33
|
+
config = if configured_accelerator.is_a?(Hash)
|
|
34
|
+
configured_accelerator.to_h.dup
|
|
35
|
+
else
|
|
36
|
+
# RUBYDB_ACCELERATOR=off|auto|required is a shorthand environment
|
|
37
|
+
# override. The environment parser may therefore replace the
|
|
38
|
+
# nested accelerator hash with a string; normalize it back into
|
|
39
|
+
# the client configuration instead of calling Hash#to_h on it.
|
|
40
|
+
{mode: configured_accelerator.to_s}
|
|
41
|
+
end
|
|
42
|
+
config[:mode] = options[:accelerator_mode] if options[:accelerator_mode]
|
|
43
|
+
client = RubyDB::Accelerator::Client.new(config)
|
|
44
|
+
details = client.stats
|
|
45
|
+
if options[:ping] && details[:available]
|
|
46
|
+
details[:ping] = client.ping
|
|
47
|
+
end
|
|
48
|
+
details[:healthy] = if details[:mode] == "off"
|
|
49
|
+
true
|
|
50
|
+
elsif details[:available]
|
|
51
|
+
!options[:ping] || details[:ping].is_a?(Hash)
|
|
52
|
+
else
|
|
53
|
+
details[:mode] != "required"
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
if options[:json]
|
|
57
|
+
@output.json(details)
|
|
58
|
+
else
|
|
59
|
+
@output.puts "RubyDB Go accelerator"
|
|
60
|
+
details.each { |key, value| @output.puts " #{key}: #{value.inspect}" }
|
|
61
|
+
end
|
|
62
|
+
details[:healthy] ? 0 : 1
|
|
63
|
+
rescue RubyDB::Accelerator::Error => error
|
|
64
|
+
@output.error(error.message)
|
|
65
|
+
1
|
|
66
|
+
ensure
|
|
67
|
+
client&.close
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
@@ -42,6 +42,9 @@ module RubyDB
|
|
|
42
42
|
checks << {name: "Connection", passed: true, details: {database: db_path, latency_ms: latency_ms}}
|
|
43
43
|
checks << {name: "Storage", passed: engine.storage_manager.open?, details: engine.storage_manager.stats}
|
|
44
44
|
checks << {name: "WAL", passed: !engine.wal.nil?, details: engine.wal.stats}
|
|
45
|
+
accelerator = engine.accelerator.stats
|
|
46
|
+
accelerator_passed = accelerator[:mode] != "required" || accelerator[:available]
|
|
47
|
+
checks << {name: "Go accelerator", passed: accelerator_passed, details: accelerator}
|
|
45
48
|
unless options[:quick]
|
|
46
49
|
checks << {name: "Indexes", passed: !engine.index_manager.nil?, details: engine.index_manager.stats}
|
|
47
50
|
end
|
data/lib/rubydb/client/client.rb
CHANGED
|
@@ -140,6 +140,13 @@ module RubyDB
|
|
|
140
140
|
end
|
|
141
141
|
end
|
|
142
142
|
|
|
143
|
+
def metadata(table_name = nil)
|
|
144
|
+
@lock.synchronize do
|
|
145
|
+
ensure_connected
|
|
146
|
+
@connection.send_metadata(table_name)
|
|
147
|
+
end
|
|
148
|
+
end
|
|
149
|
+
|
|
143
150
|
def query_async(sql, params = [], timeout: nil)
|
|
144
151
|
@lock.synchronize do
|
|
145
152
|
ensure_connected
|
|
@@ -150,6 +150,21 @@ module RubyDB
|
|
|
150
150
|
end
|
|
151
151
|
end
|
|
152
152
|
|
|
153
|
+
# Fetch catalog metadata over the same authenticated connection used by
|
|
154
|
+
# SQL queries. Rails adapters need this for schema discovery in
|
|
155
|
+
# client/server mode; asking the server for it avoids pretending that a
|
|
156
|
+
# RubyDB catalog is SQLite's sqlite_master table.
|
|
157
|
+
def send_metadata(table_name = nil)
|
|
158
|
+
@lock.synchronize do
|
|
159
|
+
ensure_connected
|
|
160
|
+
|
|
161
|
+
payload = {}
|
|
162
|
+
payload[:table] = table_name.to_s if table_name
|
|
163
|
+
send_message(Protocol::Message.new(:metadata, payload))
|
|
164
|
+
unwrap_result(receive_message.payload)
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
|
|
153
168
|
# Starts a query and returns a handle that can issue a wire-level
|
|
154
169
|
# cancellation while the server is executing it. The same connection
|
|
155
170
|
# must not be used for another application request until wait returns.
|
data/lib/rubydb/client/result.rb
CHANGED
|
@@ -5,7 +5,7 @@ module RubyDB
|
|
|
5
5
|
# Result - Query result set
|
|
6
6
|
class Result
|
|
7
7
|
attr_reader :columns, :rows, :row_count, :affected_rows, :sql
|
|
8
|
-
attr_reader :command_tag, :statement_id, :transaction_id
|
|
8
|
+
attr_reader :command_tag, :statement_id, :transaction_id, :row_id, :inserted_id
|
|
9
9
|
attr_reader :error
|
|
10
10
|
|
|
11
11
|
def initialize(data = {})
|
|
@@ -17,6 +17,8 @@ module RubyDB
|
|
|
17
17
|
@command_tag = data[:command_tag] || "SELECT"
|
|
18
18
|
@statement_id = data[:statement_id]
|
|
19
19
|
@transaction_id = data[:transaction_id]
|
|
20
|
+
@row_id = data[:row_id]
|
|
21
|
+
@inserted_id = data[:inserted_id]
|
|
20
22
|
@success = data[:success] != false
|
|
21
23
|
@error = data[:error]
|
|
22
24
|
@warnings = data[:warnings] || []
|
|
@@ -85,6 +87,8 @@ module RubyDB
|
|
|
85
87
|
row_count: @row_count,
|
|
86
88
|
affected_rows: @affected_rows,
|
|
87
89
|
command_tag: @command_tag,
|
|
90
|
+
row_id: @row_id,
|
|
91
|
+
inserted_id: @inserted_id,
|
|
88
92
|
success: @success,
|
|
89
93
|
error: @error,
|
|
90
94
|
execution_time_ms: @execution_time_ms
|
|
@@ -119,6 +119,18 @@ module RubyDB
|
|
|
119
119
|
debug: false,
|
|
120
120
|
profile: false,
|
|
121
121
|
fake_data: false
|
|
122
|
+
},
|
|
123
|
+
|
|
124
|
+
# Bundled Go acceleration. Ruby remains the correctness path and
|
|
125
|
+
# automatically handles operations the accelerator cannot prove safe.
|
|
126
|
+
accelerator: {
|
|
127
|
+
mode: "auto",
|
|
128
|
+
timeout: 30,
|
|
129
|
+
max_frame_size: 16 * 1024 * 1024,
|
|
130
|
+
min_rows: 256,
|
|
131
|
+
read_pipeline: "on",
|
|
132
|
+
direct_snapshot: true,
|
|
133
|
+
binary: nil
|
|
122
134
|
}
|
|
123
135
|
}
|
|
124
136
|
|
|
@@ -77,7 +77,14 @@ module RubyDB
|
|
|
77
77
|
"monitoring.health_check_interval" => {type: :integer, required: true, min: 1},
|
|
78
78
|
"monitoring.stats_collector" => {type: :boolean, required: true},
|
|
79
79
|
"monitoring.prometheus" => {type: :boolean, required: true},
|
|
80
|
-
"monitoring.prometheus_port" => {type: :integer, required: true, min: 1, max: 65535}
|
|
80
|
+
"monitoring.prometheus_port" => {type: :integer, required: true, min: 1, max: 65535},
|
|
81
|
+
|
|
82
|
+
"accelerator.mode" => {type: :string, required: true, enum: ["auto", "off", "required"]},
|
|
83
|
+
"accelerator.timeout" => {type: :integer, required: true, min: 1},
|
|
84
|
+
"accelerator.max_frame_size" => {type: :integer, required: true, min: 1024},
|
|
85
|
+
"accelerator.min_rows" => {type: :integer, required: true, min: 0},
|
|
86
|
+
"accelerator.read_pipeline" => {type: :string, required: true, enum: ["off", "on"]},
|
|
87
|
+
"accelerator.direct_snapshot" => {type: :boolean, required: true}
|
|
81
88
|
}
|
|
82
89
|
|
|
83
90
|
def initialize
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RubyDB
|
|
4
|
+
module Execution
|
|
5
|
+
# Keeps capability checks and result validation outside the executor's SQL
|
|
6
|
+
# logic. The Go path is an optimization; this object never makes it the
|
|
7
|
+
# source of truth.
|
|
8
|
+
class AcceleratorDispatch
|
|
9
|
+
def initialize(accelerator)
|
|
10
|
+
@accelerator = accelerator
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def preferred?(workload)
|
|
14
|
+
@accelerator&.manager&.acceleration_preferred?(workload)
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def equivalent?(left, right)
|
|
18
|
+
normalize(left) == normalize(right)
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
private
|
|
22
|
+
|
|
23
|
+
def normalize(rows)
|
|
24
|
+
rows.map do |row|
|
|
25
|
+
row.each_with_object({}) { |(key, value), normalized| normalized[key.to_s] = value }
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RubyDB
|
|
4
|
+
module Execution
|
|
5
|
+
# Conservative planner estimates. They are deliberately independent from
|
|
6
|
+
# the Go runtime: a wrong estimate must never change SQL semantics.
|
|
7
|
+
class CostModel
|
|
8
|
+
attr_reader :engine
|
|
9
|
+
|
|
10
|
+
def initialize(engine)
|
|
11
|
+
@engine = engine
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def apply(plan)
|
|
15
|
+
row_count = table_row_count(plan.table_name)
|
|
16
|
+
selectivity = estimate_selectivity(plan.predicate)
|
|
17
|
+
estimated_rows = [(row_count * selectivity).ceil, 0].max
|
|
18
|
+
cost = case plan.scan_type
|
|
19
|
+
when :index
|
|
20
|
+
plan.predicate ? estimated_rows * 0.1 : row_count * 0.5
|
|
21
|
+
when :sequential
|
|
22
|
+
row_count
|
|
23
|
+
else
|
|
24
|
+
row_count
|
|
25
|
+
end
|
|
26
|
+
cost += sort_cost(estimated_rows) if plan.order_by&.any?
|
|
27
|
+
cost += estimated_rows if plan.aggregates&.any?
|
|
28
|
+
plan.set_cost(cost, estimated_rows)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def estimate_selectivity(predicate)
|
|
32
|
+
return 1.0 unless predicate
|
|
33
|
+
|
|
34
|
+
case predicate
|
|
35
|
+
when Predicate::Comparison
|
|
36
|
+
case predicate.operator
|
|
37
|
+
when :EQ then 0.01
|
|
38
|
+
when :NE then 0.9
|
|
39
|
+
when :LT, :LTE, :GT, :GTE then 0.5
|
|
40
|
+
else 0.5
|
|
41
|
+
end
|
|
42
|
+
when Predicate::And
|
|
43
|
+
estimate_selectivity(predicate.left) * estimate_selectivity(predicate.right)
|
|
44
|
+
when Predicate::Or
|
|
45
|
+
left = estimate_selectivity(predicate.left)
|
|
46
|
+
right = estimate_selectivity(predicate.right)
|
|
47
|
+
left + right - (left * right)
|
|
48
|
+
when Predicate::Not
|
|
49
|
+
1.0 - estimate_selectivity(predicate.operand)
|
|
50
|
+
else
|
|
51
|
+
0.5
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
private
|
|
56
|
+
|
|
57
|
+
def table_row_count(table_name)
|
|
58
|
+
return 0 unless table_name
|
|
59
|
+
|
|
60
|
+
Integer(@engine.table_row_count(table_name))
|
|
61
|
+
rescue
|
|
62
|
+
0
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def sort_cost(rows)
|
|
66
|
+
return 0 if rows < 2
|
|
67
|
+
|
|
68
|
+
rows * Math.log2(rows)
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|