rubydb 0.1.5 → 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +4 -0
  3. data/CHANGELOG.md +14 -0
  4. data/Gemfile.lock +1 -1
  5. data/README.md +296 -227
  6. data/Rakefile +6 -1
  7. data/accelerator/bin/SHA256SUMS +6 -0
  8. data/accelerator/bin/rubydb-accelerator-darwin-amd64 +0 -0
  9. data/accelerator/bin/rubydb-accelerator-darwin-arm64 +0 -0
  10. data/accelerator/bin/rubydb-accelerator-linux-amd64 +0 -0
  11. data/accelerator/bin/rubydb-accelerator-linux-arm64 +0 -0
  12. data/accelerator/bin/rubydb-accelerator-windows-amd64.exe +0 -0
  13. data/accelerator/bin/rubydb-accelerator-windows-arm64.exe +0 -0
  14. data/accelerator/cmd/rubydb-accelerator/main.go +11 -0
  15. data/accelerator/go.mod +3 -0
  16. data/accelerator/internal/execution/aggregate.go +94 -0
  17. data/accelerator/internal/execution/distinct.go +22 -0
  18. data/accelerator/internal/execution/filter.go +73 -0
  19. data/accelerator/internal/execution/join.go +79 -0
  20. data/accelerator/internal/execution/operators.go +167 -0
  21. data/accelerator/internal/execution/scan.go +20 -0
  22. data/accelerator/internal/execution/sort.go +62 -0
  23. data/accelerator/internal/execution/types.go +136 -0
  24. data/accelerator/internal/execution/value.go +67 -0
  25. data/accelerator/internal/memory/arena.go +47 -0
  26. data/accelerator/internal/memory/reuse.go +22 -0
  27. data/accelerator/internal/metrics/registry.go +67 -0
  28. data/accelerator/internal/parallel/bounded_queue.go +56 -0
  29. data/accelerator/internal/parallel/scheduler.go +47 -0
  30. data/accelerator/internal/parallel/worker_pool.go +53 -0
  31. data/accelerator/internal/protocol/cancellation.go +48 -0
  32. data/accelerator/internal/protocol/columnar.go +263 -0
  33. data/accelerator/internal/protocol/frame.go +187 -0
  34. data/accelerator/internal/runtime/worker.go +521 -0
  35. data/accelerator/internal/storage/page_reader.go +81 -0
  36. data/accelerator/internal/storage/snapshot_scan.go +539 -0
  37. data/accelerator/internal/wal/checksum.go +13 -0
  38. data/accelerator/internal/wal/compression.go +41 -0
  39. data/accelerator/internal/wal/group_commit.go +24 -0
  40. data/accelerator/internal/wal/record_encoder.go +40 -0
  41. data/adapters/activerecord/README.md +8 -3
  42. data/adapters/activerecord/lib/active_record/connection_adapters/rubydb_adapter.rb +50 -42
  43. data/adapters/activerecord/rubydb-activerecord.gemspec +1 -1
  44. data/docs/README.md +3 -1
  45. data/docs/architecture/go-accelerator.md +179 -0
  46. data/docs/cli.md +24 -0
  47. data/docs/contributing/benchmarking.md +16 -0
  48. data/docs/developer/local-development.md +32 -0
  49. data/docs/release.md +2 -2
  50. data/lessons/02-local-development.md +2 -2
  51. data/lessons/04-rails-complex-apps.md +2 -2
  52. data/lessons/05-rubydb-production-server.md +2 -2
  53. data/lessons/07-hybrid-microservices.md +175 -90
  54. data/lessons/10-release-readiness.md +184 -117
  55. data/lessons/11-community-adapter.md +323 -0
  56. data/lessons/12-rails-ecommerce-pressure.md +263 -0
  57. data/lib/rubydb/accelerator/client.rb +451 -0
  58. data/lib/rubydb/accelerator/error.rb +22 -0
  59. data/lib/rubydb/accelerator/manager.rb +606 -0
  60. data/lib/rubydb/accelerator.rb +13 -0
  61. data/lib/rubydb/cli/application.rb +6 -1
  62. data/lib/rubydb/cli/commands/accelerator.rb +72 -0
  63. data/lib/rubydb/cli/commands/doctor.rb +3 -0
  64. data/lib/rubydb/client/client.rb +7 -0
  65. data/lib/rubydb/client/connection.rb +15 -0
  66. data/lib/rubydb/client/result.rb +5 -1
  67. data/lib/rubydb/configuration/defaults.rb +12 -0
  68. data/lib/rubydb/configuration/validation.rb +8 -1
  69. data/lib/rubydb/execution/accelerator_dispatch.rb +30 -0
  70. data/lib/rubydb/execution/cost_model.rb +72 -0
  71. data/lib/rubydb/execution/executor.rb +373 -11
  72. data/lib/rubydb/execution/operator_selection.rb +57 -0
  73. data/lib/rubydb/execution/physical_plan.rb +47 -0
  74. data/lib/rubydb/execution/planner.rb +12 -46
  75. data/lib/rubydb/execution/sort_executor.rb +22 -8
  76. data/lib/rubydb/indexes/btree.rb +31 -2
  77. data/lib/rubydb/rubydb.rb +7 -1
  78. data/lib/rubydb/server/session.rb +45 -0
  79. data/lib/rubydb/storage/engine.rb +74 -12
  80. data/lib/rubydb/storage/snapshot_reader.rb +167 -0
  81. data/lib/rubydb/version.rb +1 -1
  82. data/lib/rubydb/wal/archive.rb +17 -0
  83. data/lib/rubydb/wal/wal.rb +1 -0
  84. data/rubydb.gemspec +12 -2
  85. data/scripts/build_accelerator +49 -0
  86. data/scripts/release +34 -4
  87. data/scripts/replication_failover_drill +2 -2
  88. metadata +49 -1
@@ -0,0 +1,40 @@
1
+ package wal
2
+
3
+ import (
4
+ "encoding/binary"
5
+ "hash/crc32"
6
+ )
7
+
8
+ // EncodeRecord provides a versioned physical record format for future WAL
9
+ // batching. Ruby remains responsible for LSN allocation and commit ordering.
10
+ // Layout: magic(4), version(1), kind(1), LSN(8), transaction(8), length(4),
11
+ // payload, CRC32(4).
12
+ var recordMagic = [4]byte{'R', 'D', 'W', 'L'}
13
+
14
+ func EncodeRecord(kind byte, lsn, transaction uint64, payload []byte) []byte {
15
+ const headerSize = 26
16
+ output := make([]byte, headerSize+len(payload)+4)
17
+ copy(output[:4], recordMagic[:])
18
+ output[4] = 1
19
+ output[5] = kind
20
+ binary.LittleEndian.PutUint64(output[6:14], lsn)
21
+ binary.LittleEndian.PutUint64(output[14:22], transaction)
22
+ binary.LittleEndian.PutUint32(output[22:26], uint32(len(payload)))
23
+ copy(output[26:], payload)
24
+ checksum := crc32.ChecksumIEEE(output[:26+len(payload)])
25
+ binary.LittleEndian.PutUint32(output[26+len(payload):], checksum)
26
+ return output
27
+ }
28
+
29
+ func VerifyRecord(record []byte) bool {
30
+ if len(record) < 30 || string(record[:4]) != string(recordMagic[:]) || record[4] != 1 {
31
+ return false
32
+ }
33
+ length := int(binary.LittleEndian.Uint32(record[22:26]))
34
+ if length < 0 || len(record) != 30+length {
35
+ return false
36
+ }
37
+ want := binary.LittleEndian.Uint32(record[len(record)-4:])
38
+ got := crc32.ChecksumIEEE(record[:len(record)-4])
39
+ return got == want
40
+ }
@@ -7,7 +7,7 @@ This gem is an adapter to RubyDB's SQL engine. It is not a PostgreSQL, MySQL, or
7
7
  ## Requirements
8
8
 
9
9
  - Ruby 3.3 or newer
10
- - RubyDB 0.1.x
10
+ - RubyDB 0.1.6 or another compatible `0.1.x` release
11
11
  - ActiveRecord 7.1, 7.2, or 8.0 (the dependency range is `>= 7.1`, `< 8.1`)
12
12
 
13
13
  ## Install
@@ -15,8 +15,8 @@ This gem is an adapter to RubyDB's SQL engine. It is not a PostgreSQL, MySQL, or
15
15
  Add both gems to the Rails application's `Gemfile`:
16
16
 
17
17
  ```ruby
18
- gem "rubydb"
19
- gem "rubydb-activerecord"
18
+ gem "rubydb", "~> 0.1.6"
19
+ gem "rubydb-activerecord", "~> 0.1.3"
20
20
  ```
21
21
 
22
22
  Then run:
@@ -25,6 +25,11 @@ Then run:
25
25
  bundle install
26
26
  ```
27
27
 
28
+ The core gem contains the optional Go accelerator and its platform binaries;
29
+ the adapter does not require Go to be installed. RubyDB remains the SQL,
30
+ transaction, WAL, and durability authority while the adapter exposes the
31
+ Rails connection, query, migration, schema, and pool APIs.
32
+
28
33
  The adapter registers itself under the `rubydb` adapter name when it is
29
34
  required by Bundler. For a manually loaded application, require it explicitly:
30
35
 
@@ -128,16 +128,15 @@ module ActiveRecord
128
128
  def primary_key(table_name)
129
129
  return @connection.engine.table_columns(table_name).find(&:primary_key?)&.name&.to_s || "id" if embedded?
130
130
 
131
- result = execute("PRAGMA table_info(#{quote_table_name(table_name)})")
132
- row = result.find { |r| r["pk"] == 1 }
133
- row ? row["name"] : "id"
131
+ remote_metadata(table_name)[:columns].find do |column|
132
+ column[:primary_key] || column["primary_key"]
133
+ end&.then { |column| (column[:name] || column["name"]).to_s } || "id"
134
134
  end
135
135
 
136
136
  def tables
137
137
  return @connection.engine.list_tables.map(&:to_s) if embedded?
138
138
 
139
- result = execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'")
140
- result.map { |row| row["name"] }
139
+ Array(remote_metadata[:tables]).map(&:to_s)
141
140
  end
142
141
 
143
142
  def table_exists?(table_name)
@@ -148,21 +147,15 @@ module ActiveRecord
148
147
  # instantiated. The embedded engine has catalog metadata already, so
149
148
  # avoid generating unsupported SQLite catalog queries.
150
149
  def data_sources
151
- return tables if embedded?
152
-
153
- super
150
+ tables
154
151
  end
155
152
 
156
153
  def data_source_exists?(name)
157
- return table_exists?(name) if embedded?
158
-
159
- super
154
+ table_exists?(name)
160
155
  end
161
156
 
162
157
  def views
163
- return [] if embedded?
164
-
165
- super
158
+ []
166
159
  end
167
160
 
168
161
  def indexes(table_name)
@@ -177,30 +170,35 @@ module ActiveRecord
177
170
  end
178
171
  end
179
172
 
180
- result = execute("SELECT name, sql FROM sqlite_master WHERE type='index' AND tbl_name=?", [table_name])
181
- result.map do |row|
182
- {
183
- name: row["name"],
184
- columns: parse_index_columns(row["sql"]),
185
- unique: row["sql"].include?("UNIQUE")
186
- }
173
+ Array(remote_metadata(table_name)[:indexes]).map do |index|
174
+ ActiveRecord::ConnectionAdapters::IndexDefinition.new(
175
+ table_name.to_s,
176
+ index[:name] || index["name"],
177
+ index[:unique] || index["unique"],
178
+ index[:columns] || index["columns"] || []
179
+ )
187
180
  end
188
181
  end
189
182
 
190
183
  def columns(table_name)
191
184
  return embedded_columns(table_name) if embedded?
192
185
 
193
- result = execute("PRAGMA table_info(#{quote_table_name(table_name)})")
194
- result.map do |row|
186
+ remote_columns(table_name)
187
+ end
188
+
189
+ def remote_columns(table_name)
190
+ Array(remote_metadata(table_name)[:columns]).map do |row|
191
+ type = row[:type] || row["type"]
192
+ nullable = row.key?(:nullable) ? row[:nullable] : row["nullable"]
195
193
  ActiveRecord::ConnectionAdapters::Column.new(
196
- row["name"],
197
- row["default"],
198
- RubyDB::Rails::Type.to_rails(row["type"]),
199
- {
200
- null: row["notnull"] == 0,
201
- primary_key: row["pk"] == 1,
202
- limit: extract_limit(row["type"])
203
- }
194
+ row[:name] || row["name"],
195
+ rails_default_value(row.key?(:default) ? row[:default] : row["default"]),
196
+ ActiveRecord::ConnectionAdapters::SqlTypeMetadata.new(
197
+ sql_type: type.to_s.upcase,
198
+ type: rails_type_for(type),
199
+ limit: extract_limit(type.to_s)
200
+ ),
201
+ nullable
204
202
  )
205
203
  end
206
204
  end
@@ -466,23 +464,25 @@ module ActiveRecord
466
464
  end
467
465
 
468
466
  def foreign_keys(table_name)
469
- return super unless embedded?
470
-
471
- constraints = @connection.engine.table_metadata[table_name.to_s]&.fetch(:constraints, []) || []
467
+ constraints = if embedded?
468
+ @connection.engine.table_metadata[table_name.to_s]&.fetch(:constraints, []) || []
469
+ else
470
+ Array(remote_metadata(table_name)[:constraints])
471
+ end
472
472
  constraints.filter_map do |constraint|
473
- type = constraint[:type] || constraint["type"]
474
- next unless type.to_s.upcase == "FOREIGN_KEY"
473
+ definition = constraint.transform_keys(&:to_sym)
474
+ next unless definition[:type].to_s.downcase.include?("foreign")
475
475
 
476
- columns = constraint[:columns] || constraint["columns"] || []
477
- reference_table = constraint[:reference_table] || constraint["reference_table"]
478
- reference_columns = constraint[:reference_columns] || constraint["reference_columns"] || ["id"]
476
+ columns = definition[:columns] || []
477
+ reference_table = definition[:reference_table]
478
+ reference_columns = definition[:reference_columns] || ["id"]
479
479
  options = {
480
480
  column: Array(columns).first.to_s,
481
481
  primary_key: Array(reference_columns).first.to_s,
482
- name: constraint[:name] || constraint["name"]
482
+ name: definition[:name]
483
483
  }
484
- options[:on_delete] = (constraint[:on_delete] || constraint["on_delete"]).to_s if constraint[:on_delete] || constraint["on_delete"]
485
- options[:on_update] = (constraint[:on_update] || constraint["on_update"]).to_s if constraint[:on_update] || constraint["on_update"]
484
+ options[:on_delete] = definition[:on_delete].to_s if definition[:on_delete]
485
+ options[:on_update] = definition[:on_update].to_s if definition[:on_update]
486
486
  ActiveRecord::ConnectionAdapters::ForeignKeyDefinition.new(table_name.to_s, reference_table.to_s, options)
487
487
  end
488
488
  end
@@ -725,6 +725,12 @@ module ActiveRecord
725
725
  !@connection.engine.nil?
726
726
  end
727
727
 
728
+ def remote_metadata(table_name = nil)
729
+ raw = @connection.client.metadata(table_name)
730
+ metadata = raw[:metadata] || raw["metadata"] || raw
731
+ metadata.transform_keys(&:to_sym)
732
+ end
733
+
728
734
  def embedded_columns(table_name)
729
735
  @connection.engine.table_columns(table_name).map do |column|
730
736
  ActiveRecord::ConnectionAdapters::Column.new(
@@ -759,6 +765,8 @@ module ActiveRecord
759
765
  # persists typed defaults, so serialize scalar defaults at this boundary
760
766
  # and let ActiveRecord cast them through the column type map.
761
767
  def rails_default_value(value)
768
+ return nil if value.nil?
769
+
762
770
  # RubyDB exposes SQL defaults as AST literals. ActiveRecord expects
763
771
  # the scalar payload when it builds its Column metadata; calling
764
772
  # `to_s` on the wrapper would leak the Ruby object inspection into
@@ -2,7 +2,7 @@
2
2
 
3
3
  Gem::Specification.new do |spec|
4
4
  spec.name = "rubydb-activerecord"
5
- spec.version = "0.1.2"
5
+ spec.version = "0.1.3"
6
6
  spec.authors = ["Aldane Hutchinson"]
7
7
  spec.email = ["aldanehutchinson5@gmail.com"]
8
8
 
data/docs/README.md CHANGED
@@ -27,13 +27,15 @@ Start here:
27
27
  - [Workload testing](operations/workload-testing.md)
28
28
  - [Release checklist](release.md)
29
29
  - [Lessons learned](lessons-learned.md)
30
- - [Ten-lesson production journey](../lessons/01-foundations.md)
30
+ - [Production journey](../lessons/01-foundations.md)
31
+ - [Build a community adapter](../lessons/11-community-adapter.md)
31
32
 
32
33
  Architecture and development:
33
34
 
34
35
  - [Current-state audit](architecture/current-state.md)
35
36
  - [Production roadmap](architecture/production-roadmap.md)
36
37
  - [Architecture overview](architecture/overview.md)
38
+ - [Ruby + Go accelerator](architecture/go-accelerator.md)
37
39
  - [Storage engine](architecture/storage-engine.md)
38
40
  - [Transactions](architecture/transactions.md)
39
41
  - [MVCC](architecture/mvcc.md)
@@ -0,0 +1,179 @@
1
+ # Ruby + Go accelerator
2
+
3
+ RubyDB installs as one Ruby package. Release gems include CGO-free Go
4
+ executables for Windows amd64/arm64, Linux amd64/arm64, and macOS amd64/arm64. A
5
+ developer needs Go only when building RubyDB itself or publishing a release;
6
+ an application developer or production operator does not need Go installed.
7
+
8
+ Ruby remains the database authority. It owns SQL meaning, physical-plan
9
+ selection, transactions, MVCC visibility, locks, schema and constraints,
10
+ permissions, WAL, recovery, and every commit decision. Go receives immutable
11
+ read batches and returns data; it never writes database pages or decides what
12
+ is committed.
13
+
14
+ ## Runtime boundary
15
+
16
+ The Ruby process starts one long-lived private worker over stdin/stdout. The
17
+ worker does not listen on a TCP port and is not reachable by application
18
+ clients.
19
+
20
+ ```text
21
+ application -> RubyDB parser/planner -> Ruby transaction and visibility checks
22
+ \-> Go binary read worker
23
+ Ruby storage/WAL/MVCC/commit <--------- Ruby remains authoritative
24
+ ```
25
+
26
+ The Go source is deliberately split by responsibility:
27
+
28
+ ```text
29
+ accelerator/cmd/rubydb-accelerator/main.go process entrypoint only
30
+ accelerator/internal/runtime/ request loop and dispatch
31
+ accelerator/internal/protocol/ bounded frames and columnar batches
32
+ accelerator/internal/execution/ filters, sorting, aggregates, joins
33
+ accelerator/internal/storage/ immutable snapshot page reads
34
+ accelerator/internal/wal/ checksums, compression, record batches
35
+ accelerator/internal/parallel/ bounded queues and worker scheduling
36
+ accelerator/internal/memory/ reusable arena and byte buffers
37
+ accelerator/internal/metrics/ operation counts and p95 timings
38
+ ```
39
+
40
+ Each package has focused tests. The storage reader accepts only a page size,
41
+ page list, and an immutable snapshot contract supplied by RubyDB; it does not
42
+ open the catalog, WAL, or mutable page state by itself. This boundary is
43
+ intentional: adding more Go code must not silently create a second transaction
44
+ or recovery authority.
45
+
46
+ Control messages use bounded JSON-lines. Large row operations use a versioned
47
+ `RDBB` binary frame with a columnar batch: column names are sent once and
48
+ values use typed, length-prefixed fields for NULL, booleans, integers, floats,
49
+ strings, bytes, and JSON fallback values. Responses contain bounded metadata
50
+ and one or more columnar result batches. This removes the old JSON/base64 row
51
+ copy from the hot path while retaining a simple control protocol.
52
+
53
+ Every frame has a request ID, protocol/version fields, a maximum size, and a
54
+ structured error status. Startup verifies the binary against `SHA256SUMS`.
55
+ Timeout, EOF, protocol mismatch, checksum failure, malformed data, or worker
56
+ exit stops that worker. In `auto`, Ruby retries the operation on the Ruby
57
+ executor; in `required`, the error is surfaced to the caller.
58
+
59
+ ## Accelerated work
60
+
61
+ The current safe physical operators are:
62
+
63
+ - immutable snapshot page and record decoding for eligible read-only scans;
64
+ - immutable B-tree entry filtering for eligible indexed scans;
65
+ - columnar serialization and deserialization;
66
+ - deterministic parallel filtering for large batches;
67
+ - filtering, projection, ordering, DISTINCT, and NULL-aware comparisons;
68
+ - grouped `COUNT`, `SUM`, `AVG`, `MIN`, and `MAX`;
69
+ - validated inner hash and merge joins;
70
+ - portable `ROW_NUMBER`, `RANK`, `DENSE_RANK`, `LAG`, `LEAD`, and aggregate window operations;
71
+ - SHA-256 checksums; and
72
+ - gzip archive compression/decompression;
73
+ - versioned WAL batch encoding, checksums, and optional compression; and
74
+ - request-scoped cancellation, multiplexed requests, and bounded JSON result batches.
75
+
76
+ WAL acceleration is deliberately a preparation boundary: Ruby assigns the
77
+ transaction ID, LSN, commit order, and commit decision. Go returns an encoded
78
+ and checksummed batch; Ruby is still responsible for writing the approved
79
+ bytes, calling `fsync`, and publishing the durable acknowledgement. A failed
80
+ or uncertain acknowledgement must remain a Ruby recovery error.
81
+
82
+ The SQL executor delegates only plans it can describe with simple identifiers,
83
+ literal predicates, and an eligible immutable read snapshot. Outer joins,
84
+ complex expressions, writes, active transactions, and visibility-sensitive
85
+ work stay in Ruby. The general row pipeline can also receive explicit
86
+ projection, DISTINCT, HAVING, window, and merge-join specifications when the
87
+ Ruby planner has already validated their semantics. Ruby applies every stage
88
+ that was not delegated and remains the result authority through differential
89
+ validation.
90
+
91
+ ### Immutable snapshot contract
92
+
93
+ Before a snapshot scan Ruby flushes dirty pages, visibility metadata, and the
94
+ table catalog while holding the engine lock. The default direct path records
95
+ the page size/count, table page lists, schema, B-tree entries, and visibility
96
+ exclusions in a manifest, then lets Go read the already-flushed database file
97
+ while that lock remains held. This avoids copying and hashing the complete
98
+ database for every read. `RUBYDB_ACCELERATOR_DIRECT_SNAPSHOT=off` (or
99
+ `accelerator.direct_snapshot: false`) selects the detached fallback: Ruby
100
+ copies the database to a short-lived private snapshot, records a SHA-256
101
+ digest, and removes the copy after the request. Go verifies the file size,
102
+ optional digest, page headers, record bounds, record flags, column count, and
103
+ type lengths before returning rows.
104
+
105
+ This path is intentionally conservative. Ruby declines it while the current
106
+ thread has a transaction or any transaction is active. Go does not read WAL,
107
+ catalog files, live indexes, or MVCC state and cannot publish writes. If a
108
+ manifest, page, index entry, type, or result is invalid, `auto` falls back to
109
+ the Ruby executor and `required` reports the error. The B-tree data is a
110
+ consistent in-memory index snapshot because RubyDB rebuilds its indexes from
111
+ metadata on open; it is not a new persisted index format.
112
+
113
+ ## Automatic selection
114
+
115
+ The default is:
116
+
117
+ ```yaml
118
+ accelerator:
119
+ mode: auto
120
+ read_pipeline: on
121
+ direct_snapshot: true
122
+ min_rows: 256
123
+ ```
124
+
125
+ For each eligible snapshot scan, the first automatic operation is differentially
126
+ checked against the Ruby implementation and timed. The result is recorded for
127
+ both the physical snapshot scan and the row-batch scan family. Aggregate and
128
+ join operations retain their own differential checks and timing. Utility
129
+ operations record checksum, compression, and WAL-batch samples as well.
130
+ `auto` keeps the Go operator only when it is faster and equivalent; otherwise
131
+ that family falls back to Ruby for the lifetime of the worker. This avoids
132
+ turning IPC overhead into a regression for small or already-optimized queries.
133
+ The decision is process-local and is reset on restart, so it must be validated
134
+ again after a deployment change.
135
+
136
+ `mode: required` bypasses the speed decision and is intended for accelerator
137
+ CI, release smoke tests, and deployments whose workload benchmark has already
138
+ established a win. `mode: off` disables the worker entirely.
139
+
140
+ ```sh
141
+ RUBYDB_ACCELERATOR=off rubydb doctor
142
+ RUBYDB_ACCELERATOR=required rubydb accelerator --ping --json
143
+ ```
144
+
145
+ The runtime reports capabilities, selected mode, checksum-verified binary,
146
+ calibration decisions, and the last worker error in `rubydb accelerator
147
+ --ping --json` and database statistics.
148
+
149
+ ## Building and packaging
150
+
151
+ Release builds are cross-compiled with CGO disabled:
152
+
153
+ ```sh
154
+ ruby scripts/build_accelerator
155
+ RUBYDB_ACCELERATOR_TARGETS=current ruby scripts/build_accelerator
156
+ bundle exec rake build:checksum
157
+ ```
158
+
159
+ The gemspec packages the Ruby bridge, source module, six supported binaries,
160
+ and the checksum manifest. A release preflight must run Go tests, Ruby
161
+ accelerator tests, checksum verification, the extracted-gem handshake, and a
162
+ representative workload benchmark before publishing.
163
+
164
+ ## Performance policy
165
+
166
+ Go is intended to improve workloads dominated by scans, joins, aggregation,
167
+ serialization, compression, or checksum work. It cannot promise that every
168
+ query becomes seconds instead of minutes: indexes, disk latency, query shape,
169
+ lock contention, and result size may dominate. The benchmark harness should
170
+ report p50/p95/p99 latency, throughput, rows/sec, an RSS snapshot where the
171
+ platform exposes it, worker/fallback metrics, and WAL-batch timing. CPU,
172
+ process restarts, and host-level memory should be collected by the deployment
173
+ monitor because those metrics are platform-specific. A speed claim is valid
174
+ only for the measured workload and platform.
175
+
176
+ Future shared immutable page snapshots and broader index/parallel operators
177
+ must preserve the same contract: Go may read a versioned snapshot, but it may
178
+ never mutate pages, indexes, catalog files, WAL, transaction state, or
179
+ visibility state.
data/docs/cli.md CHANGED
@@ -108,6 +108,30 @@ Use `inspect --wal` to correlate WAL growth with checkpoint/backup incidents.
108
108
  Do not edit files based on an inspection result; use the documented recovery
109
109
  procedure.
110
110
 
111
+ ## Go accelerator
112
+
113
+ The accelerator is a private companion process, not a second server. Inspect
114
+ the selected platform binary and perform a protocol handshake with:
115
+
116
+ ```sh
117
+ rubydb accelerator --ping --json
118
+ ```
119
+
120
+ Build it from a source checkout after installing Go:
121
+
122
+ ```sh
123
+ ruby scripts/build_accelerator
124
+ RUBYDB_ACCELERATOR_TARGETS=current ruby scripts/build_accelerator
125
+ ```
126
+
127
+ Use `RUBYDB_ACCELERATOR=off` to reproduce a query on Ruby only. The default
128
+ `mode: auto` uses the binary columnar protocol and calibrates each supported
129
+ workload family against Ruby, falling back automatically when Go is slower or
130
+ returns a mismatched result. `mode: required` is useful for CI differential
131
+ tests and readiness validation. See the [Go accelerator
132
+ architecture](architecture/go-accelerator.md) for modes, checksums, fallback,
133
+ and package behavior.
134
+
111
135
  ## SQL shell
112
136
 
113
137
  Open the configured local database shell:
@@ -6,9 +6,25 @@ Run deterministic and concurrent workloads from the repository root:
6
6
  RUBYDB_BENCHMARK_ITERATIONS=100 ruby -Ilib benchmarks/basic_workload.rb
7
7
  RUBYDB_SOAK_THREADS=16 RUBYDB_SOAK_OPERATIONS=2000 ruby benchmarks/concurrent_soak.rb
8
8
  RUBYDB_PRODUCTION_SOAK_CLIENTS=16 RUBYDB_PRODUCTION_SOAK_OPERATIONS=2000 ruby benchmarks/production_soak.rb
9
+ # Ruby-vs-Go read pipeline benchmark (10,000 rows minimum)
10
+ RUBYDB_ACCELERATOR_ROWS=10000 RUBYDB_ACCELERATOR_ITERATIONS=5 \
11
+ RUBYDB_ACCELERATOR_THREADS=4 RUBYDB_ACCELERATOR_REQUESTS=5 \
12
+ bundle exec ruby benchmarks/go_accelerator.rb
9
13
  ```
10
14
 
11
15
  Record commit, Ruby version, OS, CPU/memory/storage, dataset size, seed,
12
16
  throughput, p50/p95/p99 latency, WAL growth, and recovery time. Benchmarks are
13
17
  not universal capacity certification; compare against application-specific
14
18
  limits and repeat after schema or runtime changes.
19
+
20
+ The accelerator benchmark reports separate Ruby and Go timings, correctness,
21
+ CPU/RSS snapshots, concurrency errors, and worker lifecycle counters. It
22
+ restarts the worker and verifies that concurrent requests return the same rows.
23
+ A Go result is only useful if it is compared with the same rows, predicates,
24
+ ordering, and result count. The Go leg uses `mode: required`; the Ruby leg is
25
+ deliberately labeled `off` and does not invoke the worker. Set
26
+ `RUBYDB_ACCELERATOR_REQUIRE_SPEED=1` in CI only when the representative
27
+ workload has a known speed win. The runtime's `auto` policy performs an
28
+ equivalent comparison per workload family before keeping an operator enabled.
29
+ Keep the JSON output with the commit and hardware record; never use a single
30
+ laptop run as a production capacity guarantee.
@@ -11,6 +11,38 @@ appropriate when testing process boundaries. Do not open one embedded path from
11
11
  multiple processes. Use the reported fuzz seed and commit when reproducing a
12
12
  failure.
13
13
 
14
+ ## Bundled Go accelerator
15
+
16
+ The checkout includes the Go accelerator source under `accelerator/`. Go is
17
+ needed to build RubyDB, not to run an installed gem. The release gem contains
18
+ the platform binary and falls back to Ruby if a platform is not packaged. To
19
+ build the binaries from a checkout:
20
+
21
+ ```sh
22
+ ruby scripts/build_accelerator
23
+ ruby -Ilib exe/rubydb accelerator --ping --json
24
+ ```
25
+
26
+ Use Ruby-only mode while diagnosing a query difference:
27
+
28
+ ```sh
29
+ RUBYDB_ACCELERATOR=off bundle exec rspec spec/sql_compatibility_spec.rb
30
+ ```
31
+
32
+ The default `auto` mode uses the binary columnar protocol, checks result
33
+ equivalence against Ruby on the first eligible scan/aggregate/join, and keeps
34
+ Go only when it is faster. Use required mode in a controlled integration test
35
+ to ensure a deployment has the expected binary:
36
+
37
+ ```yaml
38
+ accelerator:
39
+ mode: required
40
+ min_rows: 0
41
+ ```
42
+
43
+ The Go worker is private to its Ruby parent. It is not a database server and
44
+ does not make an embedded file safe to open from multiple processes.
45
+
14
46
  Start with the [developer guide](../developer-guide.md). For failures, use a
15
47
  fresh copy, preserve WAL and metadata, run the narrowest spec first, then the
16
48
  full suite. The [debugging playbook](../debugging.md) describes safe logging,
data/docs/release.md CHANGED
@@ -28,7 +28,7 @@ version to be supplied when you are not running from a matching Git tag.
28
28
  On Windows PowerShell, run this from the repository root:
29
29
 
30
30
  ```powershell
31
- $env:RUBYDB_RELEASE_VERSION = "0.1.5"
31
+ $env:RUBYDB_RELEASE_VERSION = "0.1.6"
32
32
  $env:RUBYDB_PUBLISH = "1"
33
33
  $env:GEM_HOST_API_KEY = "YOUR_RUBYGEMS_API_KEY"
34
34
  ruby scripts/release
@@ -49,7 +49,7 @@ file automatically. Keep the file private and never commit it.
49
49
  On macOS/Linux, use:
50
50
 
51
51
  ```sh
52
- RUBYDB_RELEASE_VERSION=0.1.5 \
52
+ RUBYDB_RELEASE_VERSION=0.1.6 \
53
53
  RUBYDB_PUBLISH=1 \
54
54
  GEM_HOST_API_KEY="YOUR_RUBYGEMS_API_KEY" \
55
55
  ruby scripts/release
@@ -12,8 +12,8 @@ RubyDB project at the time this lesson was written.
12
12
 
13
13
  ```ruby
14
14
  # Gemfile
15
- gem "rubydb", "0.1.5"
16
- gem "rubydb-activerecord", "0.1.2"
15
+ gem "rubydb", "0.1.6"
16
+ gem "rubydb-activerecord", "0.1.3"
17
17
  gem "pg" # Keep this when production may use PostgreSQL.
18
18
  ```
19
19
 
@@ -9,8 +9,8 @@ topology you will deploy.
9
9
 
10
10
  ```ruby
11
11
  # Gemfile
12
- gem "rubydb", "0.1.5"
13
- gem "rubydb-activerecord", "0.1.2"
12
+ gem "rubydb", "0.1.6"
13
+ gem "rubydb-activerecord", "0.1.3"
14
14
  ```
15
15
 
16
16
  Embedded development configuration:
@@ -11,7 +11,7 @@ Use a dedicated service account and persistent storage. The commands below
11
11
  assume a Unix-like host; adapt ownership commands to your platform:
12
12
 
13
13
  ```sh
14
- gem install rubydb -v 0.1.5
14
+ gem install rubydb -v 0.1.6
15
15
  install -d -o rubydb -g rubydb -m 0700 /var/lib/rubydb/data
16
16
  install -d -o rubydb -g rubydb -m 0750 /var/log/rubydb
17
17
  install -d -o rubydb -g rubydb -m 0700 /etc/rubydb
@@ -62,7 +62,7 @@ and a private network route from the application. Run these commands as an
62
62
  administrator and replace paths only after verifying the target host:
63
63
 
64
64
  ```sh
65
- gem install rubydb -v 0.1.5
65
+ gem install rubydb -v 0.1.6
66
66
  useradd --system --home-dir /var/lib/rubydb --shell /usr/sbin/nologin rubydb
67
67
  install -d -o rubydb -g rubydb -m 0700 /var/lib/rubydb/data
68
68
  install -d -o rubydb -g rubydb -m 0750 /var/log/rubydb