rubydb 0.1.5 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +4 -0
- data/CHANGELOG.md +14 -0
- data/Gemfile.lock +1 -1
- data/README.md +296 -227
- data/Rakefile +6 -1
- data/accelerator/bin/SHA256SUMS +6 -0
- data/accelerator/bin/rubydb-accelerator-darwin-amd64 +0 -0
- data/accelerator/bin/rubydb-accelerator-darwin-arm64 +0 -0
- data/accelerator/bin/rubydb-accelerator-linux-amd64 +0 -0
- data/accelerator/bin/rubydb-accelerator-linux-arm64 +0 -0
- data/accelerator/bin/rubydb-accelerator-windows-amd64.exe +0 -0
- data/accelerator/bin/rubydb-accelerator-windows-arm64.exe +0 -0
- data/accelerator/cmd/rubydb-accelerator/main.go +11 -0
- data/accelerator/go.mod +3 -0
- data/accelerator/internal/execution/aggregate.go +94 -0
- data/accelerator/internal/execution/distinct.go +22 -0
- data/accelerator/internal/execution/filter.go +73 -0
- data/accelerator/internal/execution/join.go +79 -0
- data/accelerator/internal/execution/operators.go +167 -0
- data/accelerator/internal/execution/scan.go +20 -0
- data/accelerator/internal/execution/sort.go +62 -0
- data/accelerator/internal/execution/types.go +136 -0
- data/accelerator/internal/execution/value.go +67 -0
- data/accelerator/internal/memory/arena.go +47 -0
- data/accelerator/internal/memory/reuse.go +22 -0
- data/accelerator/internal/metrics/registry.go +67 -0
- data/accelerator/internal/parallel/bounded_queue.go +56 -0
- data/accelerator/internal/parallel/scheduler.go +47 -0
- data/accelerator/internal/parallel/worker_pool.go +53 -0
- data/accelerator/internal/protocol/cancellation.go +48 -0
- data/accelerator/internal/protocol/columnar.go +263 -0
- data/accelerator/internal/protocol/frame.go +187 -0
- data/accelerator/internal/runtime/worker.go +521 -0
- data/accelerator/internal/storage/page_reader.go +81 -0
- data/accelerator/internal/storage/snapshot_scan.go +539 -0
- data/accelerator/internal/wal/checksum.go +13 -0
- data/accelerator/internal/wal/compression.go +41 -0
- data/accelerator/internal/wal/group_commit.go +24 -0
- data/accelerator/internal/wal/record_encoder.go +40 -0
- data/adapters/activerecord/README.md +8 -3
- data/adapters/activerecord/lib/active_record/connection_adapters/rubydb_adapter.rb +50 -42
- data/adapters/activerecord/rubydb-activerecord.gemspec +1 -1
- data/docs/README.md +3 -1
- data/docs/architecture/go-accelerator.md +179 -0
- data/docs/cli.md +24 -0
- data/docs/contributing/benchmarking.md +16 -0
- data/docs/developer/local-development.md +32 -0
- data/docs/release.md +2 -2
- data/lessons/02-local-development.md +2 -2
- data/lessons/04-rails-complex-apps.md +2 -2
- data/lessons/05-rubydb-production-server.md +2 -2
- data/lessons/07-hybrid-microservices.md +175 -90
- data/lessons/10-release-readiness.md +184 -117
- data/lessons/11-community-adapter.md +323 -0
- data/lessons/12-rails-ecommerce-pressure.md +263 -0
- data/lib/rubydb/accelerator/client.rb +451 -0
- data/lib/rubydb/accelerator/error.rb +22 -0
- data/lib/rubydb/accelerator/manager.rb +606 -0
- data/lib/rubydb/accelerator.rb +13 -0
- data/lib/rubydb/cli/application.rb +6 -1
- data/lib/rubydb/cli/commands/accelerator.rb +72 -0
- data/lib/rubydb/cli/commands/doctor.rb +3 -0
- data/lib/rubydb/client/client.rb +7 -0
- data/lib/rubydb/client/connection.rb +15 -0
- data/lib/rubydb/client/result.rb +5 -1
- data/lib/rubydb/configuration/defaults.rb +12 -0
- data/lib/rubydb/configuration/validation.rb +8 -1
- data/lib/rubydb/execution/accelerator_dispatch.rb +30 -0
- data/lib/rubydb/execution/cost_model.rb +72 -0
- data/lib/rubydb/execution/executor.rb +373 -11
- data/lib/rubydb/execution/operator_selection.rb +57 -0
- data/lib/rubydb/execution/physical_plan.rb +47 -0
- data/lib/rubydb/execution/planner.rb +12 -46
- data/lib/rubydb/execution/sort_executor.rb +22 -8
- data/lib/rubydb/indexes/btree.rb +31 -2
- data/lib/rubydb/rubydb.rb +7 -1
- data/lib/rubydb/server/session.rb +45 -0
- data/lib/rubydb/storage/engine.rb +74 -12
- data/lib/rubydb/storage/snapshot_reader.rb +167 -0
- data/lib/rubydb/version.rb +1 -1
- data/lib/rubydb/wal/archive.rb +17 -0
- data/lib/rubydb/wal/wal.rb +1 -0
- data/rubydb.gemspec +12 -2
- data/scripts/build_accelerator +49 -0
- data/scripts/release +34 -4
- data/scripts/replication_failover_drill +2 -2
- metadata +49 -1
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
package wal
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"encoding/binary"
|
|
5
|
+
"hash/crc32"
|
|
6
|
+
)
|
|
7
|
+
|
|
8
|
+
// EncodeRecord provides a versioned physical record format for future WAL
|
|
9
|
+
// batching. Ruby remains responsible for LSN allocation and commit ordering.
|
|
10
|
+
// Layout: magic(4), version(1), kind(1), LSN(8), transaction(8), length(4),
|
|
11
|
+
// payload, CRC32(4).
|
|
12
|
+
var recordMagic = [4]byte{'R', 'D', 'W', 'L'}
|
|
13
|
+
|
|
14
|
+
func EncodeRecord(kind byte, lsn, transaction uint64, payload []byte) []byte {
|
|
15
|
+
const headerSize = 26
|
|
16
|
+
output := make([]byte, headerSize+len(payload)+4)
|
|
17
|
+
copy(output[:4], recordMagic[:])
|
|
18
|
+
output[4] = 1
|
|
19
|
+
output[5] = kind
|
|
20
|
+
binary.LittleEndian.PutUint64(output[6:14], lsn)
|
|
21
|
+
binary.LittleEndian.PutUint64(output[14:22], transaction)
|
|
22
|
+
binary.LittleEndian.PutUint32(output[22:26], uint32(len(payload)))
|
|
23
|
+
copy(output[26:], payload)
|
|
24
|
+
checksum := crc32.ChecksumIEEE(output[:26+len(payload)])
|
|
25
|
+
binary.LittleEndian.PutUint32(output[26+len(payload):], checksum)
|
|
26
|
+
return output
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
func VerifyRecord(record []byte) bool {
|
|
30
|
+
if len(record) < 30 || string(record[:4]) != string(recordMagic[:]) || record[4] != 1 {
|
|
31
|
+
return false
|
|
32
|
+
}
|
|
33
|
+
length := int(binary.LittleEndian.Uint32(record[22:26]))
|
|
34
|
+
if length < 0 || len(record) != 30+length {
|
|
35
|
+
return false
|
|
36
|
+
}
|
|
37
|
+
want := binary.LittleEndian.Uint32(record[len(record)-4:])
|
|
38
|
+
got := crc32.ChecksumIEEE(record[:len(record)-4])
|
|
39
|
+
return got == want
|
|
40
|
+
}
|
|
@@ -7,7 +7,7 @@ This gem is an adapter to RubyDB's SQL engine. It is not a PostgreSQL, MySQL, or
|
|
|
7
7
|
## Requirements
|
|
8
8
|
|
|
9
9
|
- Ruby 3.3 or newer
|
|
10
|
-
- RubyDB 0.1.x
|
|
10
|
+
- RubyDB 0.1.6 or another compatible `0.1.x` release
|
|
11
11
|
- ActiveRecord 7.1, 7.2, or 8.0 (the dependency range is `>= 7.1`, `< 8.1`)
|
|
12
12
|
|
|
13
13
|
## Install
|
|
@@ -15,8 +15,8 @@ This gem is an adapter to RubyDB's SQL engine. It is not a PostgreSQL, MySQL, or
|
|
|
15
15
|
Add both gems to the Rails application's `Gemfile`:
|
|
16
16
|
|
|
17
17
|
```ruby
|
|
18
|
-
gem "rubydb"
|
|
19
|
-
gem "rubydb-activerecord"
|
|
18
|
+
gem "rubydb", "~> 0.1.6"
|
|
19
|
+
gem "rubydb-activerecord", "~> 0.1.3"
|
|
20
20
|
```
|
|
21
21
|
|
|
22
22
|
Then run:
|
|
@@ -25,6 +25,11 @@ Then run:
|
|
|
25
25
|
bundle install
|
|
26
26
|
```
|
|
27
27
|
|
|
28
|
+
The core gem contains the optional Go accelerator and its platform binaries;
|
|
29
|
+
the adapter does not require Go to be installed. RubyDB remains the SQL,
|
|
30
|
+
transaction, WAL, and durability authority while the adapter exposes the
|
|
31
|
+
Rails connection, query, migration, schema, and pool APIs.
|
|
32
|
+
|
|
28
33
|
The adapter registers itself under the `rubydb` adapter name when it is
|
|
29
34
|
required by Bundler. For a manually loaded application, require it explicitly:
|
|
30
35
|
|
|
@@ -128,16 +128,15 @@ module ActiveRecord
|
|
|
128
128
|
def primary_key(table_name)
|
|
129
129
|
return @connection.engine.table_columns(table_name).find(&:primary_key?)&.name&.to_s || "id" if embedded?
|
|
130
130
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
131
|
+
remote_metadata(table_name)[:columns].find do |column|
|
|
132
|
+
column[:primary_key] || column["primary_key"]
|
|
133
|
+
end&.then { |column| (column[:name] || column["name"]).to_s } || "id"
|
|
134
134
|
end
|
|
135
135
|
|
|
136
136
|
def tables
|
|
137
137
|
return @connection.engine.list_tables.map(&:to_s) if embedded?
|
|
138
138
|
|
|
139
|
-
|
|
140
|
-
result.map { |row| row["name"] }
|
|
139
|
+
Array(remote_metadata[:tables]).map(&:to_s)
|
|
141
140
|
end
|
|
142
141
|
|
|
143
142
|
def table_exists?(table_name)
|
|
@@ -148,21 +147,15 @@ module ActiveRecord
|
|
|
148
147
|
# instantiated. The embedded engine has catalog metadata already, so
|
|
149
148
|
# avoid generating unsupported SQLite catalog queries.
|
|
150
149
|
def data_sources
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
super
|
|
150
|
+
tables
|
|
154
151
|
end
|
|
155
152
|
|
|
156
153
|
def data_source_exists?(name)
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
super
|
|
154
|
+
table_exists?(name)
|
|
160
155
|
end
|
|
161
156
|
|
|
162
157
|
def views
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
super
|
|
158
|
+
[]
|
|
166
159
|
end
|
|
167
160
|
|
|
168
161
|
def indexes(table_name)
|
|
@@ -177,30 +170,35 @@ module ActiveRecord
|
|
|
177
170
|
end
|
|
178
171
|
end
|
|
179
172
|
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
name
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
173
|
+
Array(remote_metadata(table_name)[:indexes]).map do |index|
|
|
174
|
+
ActiveRecord::ConnectionAdapters::IndexDefinition.new(
|
|
175
|
+
table_name.to_s,
|
|
176
|
+
index[:name] || index["name"],
|
|
177
|
+
index[:unique] || index["unique"],
|
|
178
|
+
index[:columns] || index["columns"] || []
|
|
179
|
+
)
|
|
187
180
|
end
|
|
188
181
|
end
|
|
189
182
|
|
|
190
183
|
def columns(table_name)
|
|
191
184
|
return embedded_columns(table_name) if embedded?
|
|
192
185
|
|
|
193
|
-
|
|
194
|
-
|
|
186
|
+
remote_columns(table_name)
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
def remote_columns(table_name)
|
|
190
|
+
Array(remote_metadata(table_name)[:columns]).map do |row|
|
|
191
|
+
type = row[:type] || row["type"]
|
|
192
|
+
nullable = row.key?(:nullable) ? row[:nullable] : row["nullable"]
|
|
195
193
|
ActiveRecord::ConnectionAdapters::Column.new(
|
|
196
|
-
row["name"],
|
|
197
|
-
row["default"],
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
194
|
+
row[:name] || row["name"],
|
|
195
|
+
rails_default_value(row.key?(:default) ? row[:default] : row["default"]),
|
|
196
|
+
ActiveRecord::ConnectionAdapters::SqlTypeMetadata.new(
|
|
197
|
+
sql_type: type.to_s.upcase,
|
|
198
|
+
type: rails_type_for(type),
|
|
199
|
+
limit: extract_limit(type.to_s)
|
|
200
|
+
),
|
|
201
|
+
nullable
|
|
204
202
|
)
|
|
205
203
|
end
|
|
206
204
|
end
|
|
@@ -466,23 +464,25 @@ module ActiveRecord
|
|
|
466
464
|
end
|
|
467
465
|
|
|
468
466
|
def foreign_keys(table_name)
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
467
|
+
constraints = if embedded?
|
|
468
|
+
@connection.engine.table_metadata[table_name.to_s]&.fetch(:constraints, []) || []
|
|
469
|
+
else
|
|
470
|
+
Array(remote_metadata(table_name)[:constraints])
|
|
471
|
+
end
|
|
472
472
|
constraints.filter_map do |constraint|
|
|
473
|
-
|
|
474
|
-
next unless type.to_s.
|
|
473
|
+
definition = constraint.transform_keys(&:to_sym)
|
|
474
|
+
next unless definition[:type].to_s.downcase.include?("foreign")
|
|
475
475
|
|
|
476
|
-
columns =
|
|
477
|
-
reference_table =
|
|
478
|
-
reference_columns =
|
|
476
|
+
columns = definition[:columns] || []
|
|
477
|
+
reference_table = definition[:reference_table]
|
|
478
|
+
reference_columns = definition[:reference_columns] || ["id"]
|
|
479
479
|
options = {
|
|
480
480
|
column: Array(columns).first.to_s,
|
|
481
481
|
primary_key: Array(reference_columns).first.to_s,
|
|
482
|
-
name:
|
|
482
|
+
name: definition[:name]
|
|
483
483
|
}
|
|
484
|
-
options[:on_delete] =
|
|
485
|
-
options[:on_update] =
|
|
484
|
+
options[:on_delete] = definition[:on_delete].to_s if definition[:on_delete]
|
|
485
|
+
options[:on_update] = definition[:on_update].to_s if definition[:on_update]
|
|
486
486
|
ActiveRecord::ConnectionAdapters::ForeignKeyDefinition.new(table_name.to_s, reference_table.to_s, options)
|
|
487
487
|
end
|
|
488
488
|
end
|
|
@@ -725,6 +725,12 @@ module ActiveRecord
|
|
|
725
725
|
!@connection.engine.nil?
|
|
726
726
|
end
|
|
727
727
|
|
|
728
|
+
def remote_metadata(table_name = nil)
|
|
729
|
+
raw = @connection.client.metadata(table_name)
|
|
730
|
+
metadata = raw[:metadata] || raw["metadata"] || raw
|
|
731
|
+
metadata.transform_keys(&:to_sym)
|
|
732
|
+
end
|
|
733
|
+
|
|
728
734
|
def embedded_columns(table_name)
|
|
729
735
|
@connection.engine.table_columns(table_name).map do |column|
|
|
730
736
|
ActiveRecord::ConnectionAdapters::Column.new(
|
|
@@ -759,6 +765,8 @@ module ActiveRecord
|
|
|
759
765
|
# persists typed defaults, so serialize scalar defaults at this boundary
|
|
760
766
|
# and let ActiveRecord cast them through the column type map.
|
|
761
767
|
def rails_default_value(value)
|
|
768
|
+
return nil if value.nil?
|
|
769
|
+
|
|
762
770
|
# RubyDB exposes SQL defaults as AST literals. ActiveRecord expects
|
|
763
771
|
# the scalar payload when it builds its Column metadata; calling
|
|
764
772
|
# `to_s` on the wrapper would leak the Ruby object inspection into
|
data/docs/README.md
CHANGED
|
@@ -27,13 +27,15 @@ Start here:
|
|
|
27
27
|
- [Workload testing](operations/workload-testing.md)
|
|
28
28
|
- [Release checklist](release.md)
|
|
29
29
|
- [Lessons learned](lessons-learned.md)
|
|
30
|
-
- [
|
|
30
|
+
- [Production journey](../lessons/01-foundations.md)
|
|
31
|
+
- [Build a community adapter](../lessons/11-community-adapter.md)
|
|
31
32
|
|
|
32
33
|
Architecture and development:
|
|
33
34
|
|
|
34
35
|
- [Current-state audit](architecture/current-state.md)
|
|
35
36
|
- [Production roadmap](architecture/production-roadmap.md)
|
|
36
37
|
- [Architecture overview](architecture/overview.md)
|
|
38
|
+
- [Ruby + Go accelerator](architecture/go-accelerator.md)
|
|
37
39
|
- [Storage engine](architecture/storage-engine.md)
|
|
38
40
|
- [Transactions](architecture/transactions.md)
|
|
39
41
|
- [MVCC](architecture/mvcc.md)
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
# Ruby + Go accelerator
|
|
2
|
+
|
|
3
|
+
RubyDB installs as one Ruby package. Release gems include CGO-free Go
|
|
4
|
+
executables for Windows amd64/arm64, Linux amd64/arm64, and macOS amd64/arm64. A
|
|
5
|
+
developer needs Go only when building RubyDB itself or publishing a release;
|
|
6
|
+
an application developer or production operator does not need Go installed.
|
|
7
|
+
|
|
8
|
+
Ruby remains the database authority. It owns SQL meaning, physical-plan
|
|
9
|
+
selection, transactions, MVCC visibility, locks, schema and constraints,
|
|
10
|
+
permissions, WAL, recovery, and every commit decision. Go receives immutable
|
|
11
|
+
read batches and returns data; it never writes database pages or decides what
|
|
12
|
+
is committed.
|
|
13
|
+
|
|
14
|
+
## Runtime boundary
|
|
15
|
+
|
|
16
|
+
The Ruby process starts one long-lived private worker over stdin/stdout. The
|
|
17
|
+
worker does not listen on a TCP port and is not reachable by application
|
|
18
|
+
clients.
|
|
19
|
+
|
|
20
|
+
```text
|
|
21
|
+
application -> RubyDB parser/planner -> Ruby transaction and visibility checks
|
|
22
|
+
\-> Go binary read worker
|
|
23
|
+
Ruby storage/WAL/MVCC/commit <--------- Ruby remains authoritative
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
The Go source is deliberately split by responsibility:
|
|
27
|
+
|
|
28
|
+
```text
|
|
29
|
+
accelerator/cmd/rubydb-accelerator/main.go process entrypoint only
|
|
30
|
+
accelerator/internal/runtime/ request loop and dispatch
|
|
31
|
+
accelerator/internal/protocol/ bounded frames and columnar batches
|
|
32
|
+
accelerator/internal/execution/ filters, sorting, aggregates, joins
|
|
33
|
+
accelerator/internal/storage/ immutable snapshot page reads
|
|
34
|
+
accelerator/internal/wal/ checksums, compression, record batches
|
|
35
|
+
accelerator/internal/parallel/ bounded queues and worker scheduling
|
|
36
|
+
accelerator/internal/memory/ reusable arena and byte buffers
|
|
37
|
+
accelerator/internal/metrics/ operation counts and p95 timings
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
Each package has focused tests. The storage reader accepts only a page size,
|
|
41
|
+
page list, and an immutable snapshot contract supplied by RubyDB; it does not
|
|
42
|
+
open the catalog, WAL, or mutable page state by itself. This boundary is
|
|
43
|
+
intentional: adding more Go code must not silently create a second transaction
|
|
44
|
+
or recovery authority.
|
|
45
|
+
|
|
46
|
+
Control messages use bounded JSON-lines. Large row operations use a versioned
|
|
47
|
+
`RDBB` binary frame with a columnar batch: column names are sent once and
|
|
48
|
+
values use typed, length-prefixed fields for NULL, booleans, integers, floats,
|
|
49
|
+
strings, bytes, and JSON fallback values. Responses contain bounded metadata
|
|
50
|
+
and one or more columnar result batches. This removes the old JSON/base64 row
|
|
51
|
+
copy from the hot path while retaining a simple control protocol.
|
|
52
|
+
|
|
53
|
+
Every frame has a request ID, protocol/version fields, a maximum size, and a
|
|
54
|
+
structured error status. Startup verifies the binary against `SHA256SUMS`.
|
|
55
|
+
Timeout, EOF, protocol mismatch, checksum failure, malformed data, or worker
|
|
56
|
+
exit stops that worker. In `auto`, Ruby retries the operation on the Ruby
|
|
57
|
+
executor; in `required`, the error is surfaced to the caller.
|
|
58
|
+
|
|
59
|
+
## Accelerated work
|
|
60
|
+
|
|
61
|
+
The current safe physical operators are:
|
|
62
|
+
|
|
63
|
+
- immutable snapshot page and record decoding for eligible read-only scans;
|
|
64
|
+
- immutable B-tree entry filtering for eligible indexed scans;
|
|
65
|
+
- columnar serialization and deserialization;
|
|
66
|
+
- deterministic parallel filtering for large batches;
|
|
67
|
+
- filtering, projection, ordering, DISTINCT, and NULL-aware comparisons;
|
|
68
|
+
- grouped `COUNT`, `SUM`, `AVG`, `MIN`, and `MAX`;
|
|
69
|
+
- validated inner hash and merge joins;
|
|
70
|
+
- portable `ROW_NUMBER`, `RANK`, `DENSE_RANK`, `LAG`, `LEAD`, and aggregate window operations;
|
|
71
|
+
- SHA-256 checksums; and
|
|
72
|
+
- gzip archive compression/decompression;
|
|
73
|
+
- versioned WAL batch encoding, checksums, and optional compression; and
|
|
74
|
+
- request-scoped cancellation, multiplexed requests, and bounded JSON result batches.
|
|
75
|
+
|
|
76
|
+
WAL acceleration is deliberately a preparation boundary: Ruby assigns the
|
|
77
|
+
transaction ID, LSN, commit order, and commit decision. Go returns an encoded
|
|
78
|
+
and checksummed batch; Ruby is still responsible for writing the approved
|
|
79
|
+
bytes, calling `fsync`, and publishing the durable acknowledgement. A failed
|
|
80
|
+
or uncertain acknowledgement must remain a Ruby recovery error.
|
|
81
|
+
|
|
82
|
+
The SQL executor delegates only plans it can describe with simple identifiers,
|
|
83
|
+
literal predicates, and an eligible immutable read snapshot. Outer joins,
|
|
84
|
+
complex expressions, writes, active transactions, and visibility-sensitive
|
|
85
|
+
work stay in Ruby. The general row pipeline can also receive explicit
|
|
86
|
+
projection, DISTINCT, HAVING, window, and merge-join specifications when the
|
|
87
|
+
Ruby planner has already validated their semantics. Ruby applies every stage
|
|
88
|
+
that was not delegated and remains the result authority through differential
|
|
89
|
+
validation.
|
|
90
|
+
|
|
91
|
+
### Immutable snapshot contract
|
|
92
|
+
|
|
93
|
+
Before a snapshot scan Ruby flushes dirty pages, visibility metadata, and the
|
|
94
|
+
table catalog while holding the engine lock. The default direct path records
|
|
95
|
+
the page size/count, table page lists, schema, B-tree entries, and visibility
|
|
96
|
+
exclusions in a manifest, then lets Go read the already-flushed database file
|
|
97
|
+
while that lock remains held. This avoids copying and hashing the complete
|
|
98
|
+
database for every read. `RUBYDB_ACCELERATOR_DIRECT_SNAPSHOT=off` (or
|
|
99
|
+
`accelerator.direct_snapshot: false`) selects the detached fallback: Ruby
|
|
100
|
+
copies the database to a short-lived private snapshot, records a SHA-256
|
|
101
|
+
digest, and removes the copy after the request. Go verifies the file size,
|
|
102
|
+
optional digest, page headers, record bounds, record flags, column count, and
|
|
103
|
+
type lengths before returning rows.
|
|
104
|
+
|
|
105
|
+
This path is intentionally conservative. Ruby declines it while the current
|
|
106
|
+
thread has a transaction or any transaction is active. Go does not read WAL,
|
|
107
|
+
catalog files, live indexes, or MVCC state and cannot publish writes. If a
|
|
108
|
+
manifest, page, index entry, type, or result is invalid, `auto` falls back to
|
|
109
|
+
the Ruby executor and `required` reports the error. The B-tree data is a
|
|
110
|
+
consistent in-memory index snapshot because RubyDB rebuilds its indexes from
|
|
111
|
+
metadata on open; it is not a new persisted index format.
|
|
112
|
+
|
|
113
|
+
## Automatic selection
|
|
114
|
+
|
|
115
|
+
The default is:
|
|
116
|
+
|
|
117
|
+
```yaml
|
|
118
|
+
accelerator:
|
|
119
|
+
mode: auto
|
|
120
|
+
read_pipeline: on
|
|
121
|
+
direct_snapshot: true
|
|
122
|
+
min_rows: 256
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
For each eligible snapshot scan, the first automatic operation is differentially
|
|
126
|
+
checked against the Ruby implementation and timed. The result is recorded for
|
|
127
|
+
both the physical snapshot scan and the row-batch scan family. Aggregate and
|
|
128
|
+
join operations retain their own differential checks and timing. Utility
|
|
129
|
+
operations record checksum, compression, and WAL-batch samples as well.
|
|
130
|
+
`auto` keeps the Go operator only when it is faster and equivalent; otherwise
|
|
131
|
+
that family falls back to Ruby for the lifetime of the worker. This avoids
|
|
132
|
+
turning IPC overhead into a regression for small or already-optimized queries.
|
|
133
|
+
The decision is process-local and is reset on restart, so it must be validated
|
|
134
|
+
again after a deployment change.
|
|
135
|
+
|
|
136
|
+
`mode: required` bypasses the speed decision and is intended for accelerator
|
|
137
|
+
CI, release smoke tests, and deployments whose workload benchmark has already
|
|
138
|
+
established a win. `mode: off` disables the worker entirely.
|
|
139
|
+
|
|
140
|
+
```sh
|
|
141
|
+
RUBYDB_ACCELERATOR=off rubydb doctor
|
|
142
|
+
RUBYDB_ACCELERATOR=required rubydb accelerator --ping --json
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
The runtime reports capabilities, selected mode, checksum-verified binary,
|
|
146
|
+
calibration decisions, and the last worker error in `rubydb accelerator
|
|
147
|
+
--ping --json` and database statistics.
|
|
148
|
+
|
|
149
|
+
## Building and packaging
|
|
150
|
+
|
|
151
|
+
Release builds are cross-compiled with CGO disabled:
|
|
152
|
+
|
|
153
|
+
```sh
|
|
154
|
+
ruby scripts/build_accelerator
|
|
155
|
+
RUBYDB_ACCELERATOR_TARGETS=current ruby scripts/build_accelerator
|
|
156
|
+
bundle exec rake build:checksum
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
The gemspec packages the Ruby bridge, source module, six supported binaries,
|
|
160
|
+
and the checksum manifest. A release preflight must run Go tests, Ruby
|
|
161
|
+
accelerator tests, checksum verification, the extracted-gem handshake, and a
|
|
162
|
+
representative workload benchmark before publishing.
|
|
163
|
+
|
|
164
|
+
## Performance policy
|
|
165
|
+
|
|
166
|
+
Go is intended to improve workloads dominated by scans, joins, aggregation,
|
|
167
|
+
serialization, compression, or checksum work. It cannot promise that every
|
|
168
|
+
query becomes seconds instead of minutes: indexes, disk latency, query shape,
|
|
169
|
+
lock contention, and result size may dominate. The benchmark harness should
|
|
170
|
+
report p50/p95/p99 latency, throughput, rows/sec, an RSS snapshot where the
|
|
171
|
+
platform exposes it, worker/fallback metrics, and WAL-batch timing. CPU,
|
|
172
|
+
process restarts, and host-level memory should be collected by the deployment
|
|
173
|
+
monitor because those metrics are platform-specific. A speed claim is valid
|
|
174
|
+
only for the measured workload and platform.
|
|
175
|
+
|
|
176
|
+
Future shared immutable page snapshots and broader index/parallel operators
|
|
177
|
+
must preserve the same contract: Go may read a versioned snapshot, but it may
|
|
178
|
+
never mutate pages, indexes, catalog files, WAL, transaction state, or
|
|
179
|
+
visibility state.
|
data/docs/cli.md
CHANGED
|
@@ -108,6 +108,30 @@ Use `inspect --wal` to correlate WAL growth with checkpoint/backup incidents.
|
|
|
108
108
|
Do not edit files based on an inspection result; use the documented recovery
|
|
109
109
|
procedure.
|
|
110
110
|
|
|
111
|
+
## Go accelerator
|
|
112
|
+
|
|
113
|
+
The accelerator is a private companion process, not a second server. Inspect
|
|
114
|
+
the selected platform binary and perform a protocol handshake with:
|
|
115
|
+
|
|
116
|
+
```sh
|
|
117
|
+
rubydb accelerator --ping --json
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
Build it from a source checkout after installing Go:
|
|
121
|
+
|
|
122
|
+
```sh
|
|
123
|
+
ruby scripts/build_accelerator
|
|
124
|
+
RUBYDB_ACCELERATOR_TARGETS=current ruby scripts/build_accelerator
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
Use `RUBYDB_ACCELERATOR=off` to reproduce a query on Ruby only. The default
|
|
128
|
+
`mode: auto` uses the binary columnar protocol and calibrates each supported
|
|
129
|
+
workload family against Ruby, falling back automatically when Go is slower or
|
|
130
|
+
returns a mismatched result. `mode: required` is useful for CI differential
|
|
131
|
+
tests and readiness validation. See the [Go accelerator
|
|
132
|
+
architecture](architecture/go-accelerator.md) for modes, checksums, fallback,
|
|
133
|
+
and package behavior.
|
|
134
|
+
|
|
111
135
|
## SQL shell
|
|
112
136
|
|
|
113
137
|
Open the configured local database shell:
|
|
@@ -6,9 +6,25 @@ Run deterministic and concurrent workloads from the repository root:
|
|
|
6
6
|
RUBYDB_BENCHMARK_ITERATIONS=100 ruby -Ilib benchmarks/basic_workload.rb
|
|
7
7
|
RUBYDB_SOAK_THREADS=16 RUBYDB_SOAK_OPERATIONS=2000 ruby benchmarks/concurrent_soak.rb
|
|
8
8
|
RUBYDB_PRODUCTION_SOAK_CLIENTS=16 RUBYDB_PRODUCTION_SOAK_OPERATIONS=2000 ruby benchmarks/production_soak.rb
|
|
9
|
+
# Ruby-vs-Go read pipeline benchmark (10,000 rows minimum)
|
|
10
|
+
RUBYDB_ACCELERATOR_ROWS=10000 RUBYDB_ACCELERATOR_ITERATIONS=5 \
|
|
11
|
+
RUBYDB_ACCELERATOR_THREADS=4 RUBYDB_ACCELERATOR_REQUESTS=5 \
|
|
12
|
+
bundle exec ruby benchmarks/go_accelerator.rb
|
|
9
13
|
```
|
|
10
14
|
|
|
11
15
|
Record commit, Ruby version, OS, CPU/memory/storage, dataset size, seed,
|
|
12
16
|
throughput, p50/p95/p99 latency, WAL growth, and recovery time. Benchmarks are
|
|
13
17
|
not universal capacity certification; compare against application-specific
|
|
14
18
|
limits and repeat after schema or runtime changes.
|
|
19
|
+
|
|
20
|
+
The accelerator benchmark reports separate Ruby and Go timings, correctness,
|
|
21
|
+
CPU/RSS snapshots, concurrency errors, and worker lifecycle counters. It
|
|
22
|
+
restarts the worker and verifies that concurrent requests return the same rows.
|
|
23
|
+
A Go result is only useful if it is compared with the same rows, predicates,
|
|
24
|
+
ordering, and result count. The Go leg uses `mode: required`; the Ruby leg is
|
|
25
|
+
deliberately labeled `off` and does not invoke the worker. Set
|
|
26
|
+
`RUBYDB_ACCELERATOR_REQUIRE_SPEED=1` in CI only when the representative
|
|
27
|
+
workload has a known speed win. The runtime's `auto` policy performs an
|
|
28
|
+
equivalent comparison per workload family before keeping an operator enabled.
|
|
29
|
+
Keep the JSON output with the commit and hardware record; never use a single
|
|
30
|
+
laptop run as a production capacity guarantee.
|
|
@@ -11,6 +11,38 @@ appropriate when testing process boundaries. Do not open one embedded path from
|
|
|
11
11
|
multiple processes. Use the reported fuzz seed and commit when reproducing a
|
|
12
12
|
failure.
|
|
13
13
|
|
|
14
|
+
## Bundled Go accelerator
|
|
15
|
+
|
|
16
|
+
The checkout includes the Go accelerator source under `accelerator/`. Go is
|
|
17
|
+
needed to build RubyDB, not to run an installed gem. The release gem contains
|
|
18
|
+
the platform binary and falls back to Ruby if a platform is not packaged. To
|
|
19
|
+
build the binaries from a checkout:
|
|
20
|
+
|
|
21
|
+
```sh
|
|
22
|
+
ruby scripts/build_accelerator
|
|
23
|
+
ruby -Ilib exe/rubydb accelerator --ping --json
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
Use Ruby-only mode while diagnosing a query difference:
|
|
27
|
+
|
|
28
|
+
```sh
|
|
29
|
+
RUBYDB_ACCELERATOR=off bundle exec rspec spec/sql_compatibility_spec.rb
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
The default `auto` mode uses the binary columnar protocol, checks result
|
|
33
|
+
equivalence against Ruby on the first eligible scan/aggregate/join, and keeps
|
|
34
|
+
Go only when it is faster. Use required mode in a controlled integration test
|
|
35
|
+
to ensure a deployment has the expected binary:
|
|
36
|
+
|
|
37
|
+
```yaml
|
|
38
|
+
accelerator:
|
|
39
|
+
mode: required
|
|
40
|
+
min_rows: 0
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
The Go worker is private to its Ruby parent. It is not a database server and
|
|
44
|
+
does not make an embedded file safe to open from multiple processes.
|
|
45
|
+
|
|
14
46
|
Start with the [developer guide](../developer-guide.md). For failures, use a
|
|
15
47
|
fresh copy, preserve WAL and metadata, run the narrowest spec first, then the
|
|
16
48
|
full suite. The [debugging playbook](../debugging.md) describes safe logging,
|
data/docs/release.md
CHANGED
|
@@ -28,7 +28,7 @@ version to be supplied when you are not running from a matching Git tag.
|
|
|
28
28
|
On Windows PowerShell, run this from the repository root:
|
|
29
29
|
|
|
30
30
|
```powershell
|
|
31
|
-
$env:RUBYDB_RELEASE_VERSION = "0.1.
|
|
31
|
+
$env:RUBYDB_RELEASE_VERSION = "0.1.6"
|
|
32
32
|
$env:RUBYDB_PUBLISH = "1"
|
|
33
33
|
$env:GEM_HOST_API_KEY = "YOUR_RUBYGEMS_API_KEY"
|
|
34
34
|
ruby scripts/release
|
|
@@ -49,7 +49,7 @@ file automatically. Keep the file private and never commit it.
|
|
|
49
49
|
On macOS/Linux, use:
|
|
50
50
|
|
|
51
51
|
```sh
|
|
52
|
-
RUBYDB_RELEASE_VERSION=0.1.
|
|
52
|
+
RUBYDB_RELEASE_VERSION=0.1.6 \
|
|
53
53
|
RUBYDB_PUBLISH=1 \
|
|
54
54
|
GEM_HOST_API_KEY="YOUR_RUBYGEMS_API_KEY" \
|
|
55
55
|
ruby scripts/release
|
|
@@ -12,8 +12,8 @@ RubyDB project at the time this lesson was written.
|
|
|
12
12
|
|
|
13
13
|
```ruby
|
|
14
14
|
# Gemfile
|
|
15
|
-
gem "rubydb", "0.1.
|
|
16
|
-
gem "rubydb-activerecord", "0.1.
|
|
15
|
+
gem "rubydb", "0.1.6"
|
|
16
|
+
gem "rubydb-activerecord", "0.1.3"
|
|
17
17
|
gem "pg" # Keep this when production may use PostgreSQL.
|
|
18
18
|
```
|
|
19
19
|
|
|
@@ -11,7 +11,7 @@ Use a dedicated service account and persistent storage. The commands below
|
|
|
11
11
|
assume a Unix-like host; adapt ownership commands to your platform:
|
|
12
12
|
|
|
13
13
|
```sh
|
|
14
|
-
gem install rubydb -v 0.1.
|
|
14
|
+
gem install rubydb -v 0.1.6
|
|
15
15
|
install -d -o rubydb -g rubydb -m 0700 /var/lib/rubydb/data
|
|
16
16
|
install -d -o rubydb -g rubydb -m 0750 /var/log/rubydb
|
|
17
17
|
install -d -o rubydb -g rubydb -m 0700 /etc/rubydb
|
|
@@ -62,7 +62,7 @@ and a private network route from the application. Run these commands as an
|
|
|
62
62
|
administrator and replace paths only after verifying the target host:
|
|
63
63
|
|
|
64
64
|
```sh
|
|
65
|
-
gem install rubydb -v 0.1.
|
|
65
|
+
gem install rubydb -v 0.1.6
|
|
66
66
|
useradd --system --home-dir /var/lib/rubydb --shell /usr/sbin/nologin rubydb
|
|
67
67
|
install -d -o rubydb -g rubydb -m 0700 /var/lib/rubydb/data
|
|
68
68
|
install -d -o rubydb -g rubydb -m 0750 /var/log/rubydb
|