rubydb 0.1.4 → 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (469) hide show
  1. checksums.yaml +4 -4
  2. data/.github/PULL_REQUEST_TEMPLATE.md +15 -15
  3. data/.github/workflows/benchmark.yml +26 -26
  4. data/.github/workflows/compatibility.yml +63 -63
  5. data/.github/workflows/fuzz.yml +33 -33
  6. data/.github/workflows/lint.yml +21 -21
  7. data/.github/workflows/operations.yml +24 -24
  8. data/.github/workflows/production-validation.yml +111 -111
  9. data/.github/workflows/release.yml +77 -77
  10. data/.github/workflows/security.yml +39 -37
  11. data/.github/workflows/test.yml +26 -26
  12. data/.github/workflows/workload.yml +58 -58
  13. data/.gitignore +16 -5
  14. data/.rubocop.yml +50 -44
  15. data/.standard.yml +9 -14
  16. data/ARCHITECTURE.md +21 -21
  17. data/CHANGELOG.md +57 -27
  18. data/CODE_OF_CONDUCT.md +13 -13
  19. data/CONTRIBUTING.md +29 -29
  20. data/GOVERNANCE.md +16 -16
  21. data/Gemfile +18 -17
  22. data/Gemfile.lock +125 -71
  23. data/README.md +168 -12
  24. data/ROADMAP.md +27 -27
  25. data/Rakefile +76 -71
  26. data/SECURITY.md +54 -54
  27. data/SUPPORT.md +14 -14
  28. data/accelerator/bin/SHA256SUMS +6 -0
  29. data/accelerator/bin/rubydb-accelerator-darwin-amd64 +0 -0
  30. data/accelerator/bin/rubydb-accelerator-darwin-arm64 +0 -0
  31. data/accelerator/bin/rubydb-accelerator-linux-amd64 +0 -0
  32. data/accelerator/bin/rubydb-accelerator-linux-arm64 +0 -0
  33. data/accelerator/bin/rubydb-accelerator-windows-amd64.exe +0 -0
  34. data/accelerator/bin/rubydb-accelerator-windows-arm64.exe +0 -0
  35. data/accelerator/cmd/rubydb-accelerator/main.go +11 -0
  36. data/accelerator/go.mod +3 -0
  37. data/accelerator/internal/execution/aggregate.go +94 -0
  38. data/accelerator/internal/execution/distinct.go +22 -0
  39. data/accelerator/internal/execution/filter.go +73 -0
  40. data/accelerator/internal/execution/join.go +79 -0
  41. data/accelerator/internal/execution/operators.go +167 -0
  42. data/accelerator/internal/execution/scan.go +20 -0
  43. data/accelerator/internal/execution/sort.go +62 -0
  44. data/accelerator/internal/execution/types.go +136 -0
  45. data/accelerator/internal/execution/value.go +67 -0
  46. data/accelerator/internal/memory/arena.go +47 -0
  47. data/accelerator/internal/memory/reuse.go +22 -0
  48. data/accelerator/internal/metrics/registry.go +67 -0
  49. data/accelerator/internal/parallel/bounded_queue.go +56 -0
  50. data/accelerator/internal/parallel/scheduler.go +47 -0
  51. data/accelerator/internal/parallel/worker_pool.go +53 -0
  52. data/accelerator/internal/protocol/cancellation.go +48 -0
  53. data/accelerator/internal/protocol/columnar.go +263 -0
  54. data/accelerator/internal/protocol/frame.go +187 -0
  55. data/accelerator/internal/runtime/worker.go +521 -0
  56. data/accelerator/internal/storage/page_reader.go +81 -0
  57. data/accelerator/internal/storage/snapshot_scan.go +539 -0
  58. data/accelerator/internal/wal/checksum.go +13 -0
  59. data/accelerator/internal/wal/compression.go +41 -0
  60. data/accelerator/internal/wal/group_commit.go +24 -0
  61. data/accelerator/internal/wal/record_encoder.go +40 -0
  62. data/adapters/activerecord/Gemfile +11 -11
  63. data/adapters/activerecord/README.md +8 -3
  64. data/adapters/activerecord/lib/active_record/connection_adapters/rubydb_adapter.rb +881 -879
  65. data/adapters/activerecord/rubydb-activerecord.gemspec +21 -21
  66. data/adapters/activerecord/spec/rubydb_adapter_integration_spec.rb +143 -143
  67. data/adapters/ruby/README.md +18 -18
  68. data/adapters/sequel/README.md +11 -11
  69. data/config/monitoring/prometheus-alerts.yml +39 -39
  70. data/config/production.yml +36 -36
  71. data/docs/README.md +77 -71
  72. data/docs/architecture/concurrency.md +14 -14
  73. data/docs/architecture/current-state.md +125 -125
  74. data/docs/architecture/execution-engine.md +25 -25
  75. data/docs/architecture/go-accelerator.md +179 -0
  76. data/docs/architecture/indexes.md +19 -19
  77. data/docs/architecture/mvcc.md +19 -19
  78. data/docs/architecture/overview.md +13 -13
  79. data/docs/architecture/pages.md +11 -11
  80. data/docs/architecture/production-roadmap.md +82 -82
  81. data/docs/architecture/query-planner.md +20 -20
  82. data/docs/architecture/recovery.md +18 -18
  83. data/docs/architecture/sql-engine.md +12 -12
  84. data/docs/architecture/storage-engine.md +14 -14
  85. data/docs/architecture/transactions.md +10 -10
  86. data/docs/architecture/wal.md +28 -28
  87. data/docs/cli-cheatsheet.md +98 -98
  88. data/docs/cli.md +299 -275
  89. data/docs/contributing/architecture.md +9 -9
  90. data/docs/contributing/benchmarking.md +30 -14
  91. data/docs/contributing/development.md +16 -16
  92. data/docs/contributing/release-process.md +49 -49
  93. data/docs/contributing/testing.md +16 -16
  94. data/docs/debugging.md +229 -229
  95. data/docs/developer/branching.md +10 -10
  96. data/docs/developer/database-diff.md +10 -10
  97. data/docs/developer/local-development.md +49 -17
  98. data/docs/developer/snapshots.md +9 -9
  99. data/docs/developer/temporal-data.md +10 -10
  100. data/docs/developer-guide.md +297 -297
  101. data/docs/getting-started/first-database.md +16 -16
  102. data/docs/getting-started/first-query.md +13 -13
  103. data/docs/getting-started/installation.md +19 -19
  104. data/docs/getting-started/local-to-production.md +300 -300
  105. data/docs/getting-started/quickstart.md +17 -17
  106. data/docs/getting-started/rails.md +16 -16
  107. data/docs/hardening_backlog.md +93 -93
  108. data/docs/lessons-learned.md +112 -112
  109. data/docs/operations/backups.md +33 -33
  110. data/docs/operations/disaster-recovery.md +31 -31
  111. data/docs/operations/failover.md +30 -30
  112. data/docs/operations/monitoring.md +25 -25
  113. data/docs/operations/production-guide.md +295 -295
  114. data/docs/operations/production-runbook.md +45 -45
  115. data/docs/operations/replication.md +33 -33
  116. data/docs/operations/restore.md +6 -6
  117. data/docs/operations/runbook.md +34 -34
  118. data/docs/operations/upgrades.md +14 -14
  119. data/docs/operations/workload-testing.md +17 -17
  120. data/docs/production-readiness.md +118 -118
  121. data/docs/production_validation.md +150 -150
  122. data/docs/rails/active-record.md +11 -11
  123. data/docs/rails/compatibility-guide.md +90 -90
  124. data/docs/rails/database-yml.md +92 -92
  125. data/docs/rails/installation.md +17 -17
  126. data/docs/rails/migrations.md +17 -17
  127. data/docs/rails/production.md +82 -82
  128. data/docs/rails/troubleshooting.md +18 -18
  129. data/docs/release.md +25 -25
  130. data/docs/server/architecture.md +10 -10
  131. data/docs/server/authentication.md +10 -10
  132. data/docs/server/configuration.md +16 -16
  133. data/docs/server/connection-pooling.md +10 -10
  134. data/docs/server/deployment.md +10 -10
  135. data/docs/server/protocol.md +12 -12
  136. data/docs/sql/compatibility-guide.md +82 -82
  137. data/docs/sql/compatibility.md +39 -39
  138. data/docs/sql/data-types.md +10 -10
  139. data/docs/sql/functions.md +9 -9
  140. data/docs/sql/joins.md +9 -9
  141. data/docs/sql/operators.md +9 -9
  142. data/docs/sql/sqlite-compatibility.md +21 -21
  143. data/docs/sql/syntax.md +10 -10
  144. data/docs/sql/transactions.md +10 -10
  145. data/docs/troubleshooting.md +244 -244
  146. data/lessons/01-foundations.md +73 -0
  147. data/lessons/02-local-development.md +121 -0
  148. data/lessons/03-embedded-rubydb.md +99 -0
  149. data/lessons/04-rails-complex-apps.md +138 -0
  150. data/lessons/05-rubydb-production-server.md +237 -0
  151. data/lessons/06-postgresql-massive-apps.md +96 -0
  152. data/lessons/07-hybrid-microservices.md +179 -0
  153. data/lessons/08-migrations-backups-recovery.md +86 -0
  154. data/lessons/09-observability-security-scale.md +87 -0
  155. data/lessons/10-release-readiness.md +192 -0
  156. data/lessons/11-community-adapter.md +323 -0
  157. data/lessons/12-rails-ecommerce-pressure.md +263 -0
  158. data/lib/rubydb/accelerator/client.rb +451 -0
  159. data/lib/rubydb/accelerator/error.rb +22 -0
  160. data/lib/rubydb/accelerator/manager.rb +606 -0
  161. data/lib/rubydb/accelerator.rb +13 -0
  162. data/lib/rubydb/backup/archive.rb +332 -334
  163. data/lib/rubydb/backup/backup.rb +400 -401
  164. data/lib/rubydb/backup/incremental.rb +349 -353
  165. data/lib/rubydb/backup/restore.rb +289 -290
  166. data/lib/rubydb/backup/snapshot.rb +265 -267
  167. data/lib/rubydb/backup/verification.rb +276 -279
  168. data/lib/rubydb/branching/branch.rb +181 -181
  169. data/lib/rubydb/branching/branch_manager.rb +307 -311
  170. data/lib/rubydb/branching/branch_metadata.rb +140 -140
  171. data/lib/rubydb/branching/checkout.rb +165 -166
  172. data/lib/rubydb/branching/copy_on_write.rb +272 -272
  173. data/lib/rubydb/branching/diff.rb +137 -138
  174. data/lib/rubydb/branching/merge.rb +282 -285
  175. data/lib/rubydb/build_info.rb +15 -15
  176. data/lib/rubydb/catalog/catalog.rb +391 -391
  177. data/lib/rubydb/catalog/column.rb +112 -112
  178. data/lib/rubydb/catalog/constraint.rb +180 -180
  179. data/lib/rubydb/catalog/database.rb +184 -184
  180. data/lib/rubydb/catalog/index.rb +97 -97
  181. data/lib/rubydb/catalog/schema.rb +103 -103
  182. data/lib/rubydb/catalog/sequence.rb +90 -90
  183. data/lib/rubydb/catalog/system_catalog.rb +698 -698
  184. data/lib/rubydb/catalog/table.rb +178 -178
  185. data/lib/rubydb/catalog/trigger.rb +102 -102
  186. data/lib/rubydb/catalog/view.rb +66 -66
  187. data/lib/rubydb/cli/application.rb +168 -163
  188. data/lib/rubydb/cli/commands/accelerator.rb +72 -0
  189. data/lib/rubydb/cli/commands/backup.rb +80 -81
  190. data/lib/rubydb/cli/commands/branch.rb +72 -72
  191. data/lib/rubydb/cli/commands/checkout.rb +54 -54
  192. data/lib/rubydb/cli/commands/create.rb +58 -58
  193. data/lib/rubydb/cli/commands/diff.rb +76 -77
  194. data/lib/rubydb/cli/commands/doctor.rb +77 -74
  195. data/lib/rubydb/cli/commands/drop.rb +57 -57
  196. data/lib/rubydb/cli/commands/init.rb +101 -102
  197. data/lib/rubydb/cli/commands/inspect.rb +95 -95
  198. data/lib/rubydb/cli/commands/merge.rb +63 -63
  199. data/lib/rubydb/cli/commands/migrate.rb +62 -62
  200. data/lib/rubydb/cli/commands/restart.rb +42 -39
  201. data/lib/rubydb/cli/commands/restore.rb +121 -121
  202. data/lib/rubydb/cli/commands/shell.rb +365 -365
  203. data/lib/rubydb/cli/commands/snapshot.rb +79 -79
  204. data/lib/rubydb/cli/commands/start.rb +88 -82
  205. data/lib/rubydb/cli/commands/status.rb +96 -92
  206. data/lib/rubydb/cli/commands/stop.rb +47 -47
  207. data/lib/rubydb/cli/commands/vacuum.rb +58 -58
  208. data/lib/rubydb/cli/formatter.rb +221 -221
  209. data/lib/rubydb/cli/output.rb +168 -168
  210. data/lib/rubydb/client/client.rb +309 -304
  211. data/lib/rubydb/client/connection.rb +429 -415
  212. data/lib/rubydb/client/connection_pool.rb +168 -168
  213. data/lib/rubydb/client/connection_url.rb +96 -96
  214. data/lib/rubydb/client/prepared_statement.rb +60 -60
  215. data/lib/rubydb/client/result.rb +127 -123
  216. data/lib/rubydb/client/statement.rb +52 -52
  217. data/lib/rubydb/client/transaction.rb +130 -130
  218. data/lib/rubydb/concurrency/concurrency.rb +19 -19
  219. data/lib/rubydb/concurrency/deadlock_detector.rb +148 -150
  220. data/lib/rubydb/concurrency/latch.rb +101 -101
  221. data/lib/rubydb/concurrency/lock_graph.rb +163 -165
  222. data/lib/rubydb/concurrency/mutex.rb +181 -183
  223. data/lib/rubydb/concurrency/rw_lock.rb +180 -180
  224. data/lib/rubydb/concurrency/scheduler.rb +248 -250
  225. data/lib/rubydb/concurrency/worker_pool.rb +145 -143
  226. data/lib/rubydb/configuration/config.rb +170 -170
  227. data/lib/rubydb/configuration/defaults.rb +191 -179
  228. data/lib/rubydb/configuration/environment.rb +152 -152
  229. data/lib/rubydb/configuration/parser.rb +185 -185
  230. data/lib/rubydb/configuration/validation.rb +228 -221
  231. data/lib/rubydb/constants.rb +74 -74
  232. data/lib/rubydb/constraints/check.rb +181 -181
  233. data/lib/rubydb/constraints/constraint.rb +101 -101
  234. data/lib/rubydb/constraints/foreign_key.rb +130 -130
  235. data/lib/rubydb/constraints/not_null.rb +64 -64
  236. data/lib/rubydb/constraints/primary_key.rb +99 -99
  237. data/lib/rubydb/constraints/unique.rb +106 -108
  238. data/lib/rubydb/constraints/validator.rb +349 -350
  239. data/lib/rubydb/errors/authentication_error.rb +10 -10
  240. data/lib/rubydb/errors/authorization_error.rb +23 -23
  241. data/lib/rubydb/errors/client_error.rb +10 -10
  242. data/lib/rubydb/errors/configuration_error.rb +10 -10
  243. data/lib/rubydb/errors/connection_error.rb +10 -10
  244. data/lib/rubydb/errors/constraint_error.rb +23 -23
  245. data/lib/rubydb/errors/corruption_error.rb +10 -10
  246. data/lib/rubydb/errors/database_error.rb +10 -10
  247. data/lib/rubydb/errors/error.rb +20 -20
  248. data/lib/rubydb/errors/execution_error.rb +10 -10
  249. data/lib/rubydb/errors/parser_error.rb +10 -10
  250. data/lib/rubydb/errors/recovery_error.rb +10 -10
  251. data/lib/rubydb/errors/replication_error.rb +10 -10
  252. data/lib/rubydb/errors/server_error.rb +6 -6
  253. data/lib/rubydb/errors/storage_error.rb +10 -10
  254. data/lib/rubydb/errors/transaction_error.rb +10 -10
  255. data/lib/rubydb/execution/accelerator_dispatch.rb +30 -0
  256. data/lib/rubydb/execution/aggregate_executor.rb +134 -138
  257. data/lib/rubydb/execution/cost_model.rb +72 -0
  258. data/lib/rubydb/execution/delete_executor.rb +110 -112
  259. data/lib/rubydb/execution/distinct_executor.rb +131 -135
  260. data/lib/rubydb/execution/executor.rb +1544 -1188
  261. data/lib/rubydb/execution/expression.rb +191 -193
  262. data/lib/rubydb/execution/index_scan.rb +142 -142
  263. data/lib/rubydb/execution/insert_executor.rb +215 -217
  264. data/lib/rubydb/execution/join_executor.rb +243 -249
  265. data/lib/rubydb/execution/limit_executor.rb +83 -85
  266. data/lib/rubydb/execution/operator_selection.rb +57 -0
  267. data/lib/rubydb/execution/optimizer.rb +227 -215
  268. data/lib/rubydb/execution/physical_plan.rb +47 -0
  269. data/lib/rubydb/execution/plan.rb +355 -353
  270. data/lib/rubydb/execution/planner.rb +508 -536
  271. data/lib/rubydb/execution/predicate.rb +235 -235
  272. data/lib/rubydb/execution/scan.rb +49 -49
  273. data/lib/rubydb/execution/sequential_scan.rb +63 -63
  274. data/lib/rubydb/execution/sort_executor.rb +194 -185
  275. data/lib/rubydb/execution/update_executor.rb +160 -162
  276. data/lib/rubydb/functions/aggregate.rb +70 -70
  277. data/lib/rubydb/functions/date_functions.rb +274 -278
  278. data/lib/rubydb/functions/function.rb +85 -85
  279. data/lib/rubydb/functions/json_functions.rb +231 -215
  280. data/lib/rubydb/functions/numeric_functions.rb +346 -346
  281. data/lib/rubydb/functions/scalar.rb +52 -52
  282. data/lib/rubydb/functions/string_functions.rb +383 -383
  283. data/lib/rubydb/functions/system_functions.rb +258 -246
  284. data/lib/rubydb/history/as_of.rb +238 -238
  285. data/lib/rubydb/history/change.rb +105 -105
  286. data/lib/rubydb/history/history.rb +131 -131
  287. data/lib/rubydb/history/history_manager.rb +228 -229
  288. data/lib/rubydb/history/temporal_query.rb +202 -202
  289. data/lib/rubydb/history/timeline.rb +144 -144
  290. data/lib/rubydb/indexes/btree.rb +215 -186
  291. data/lib/rubydb/indexes/btree_cursor.rb +258 -258
  292. data/lib/rubydb/indexes/btree_node.rb +384 -385
  293. data/lib/rubydb/indexes/hash_index.rb +150 -150
  294. data/lib/rubydb/indexes/index.rb +71 -71
  295. data/lib/rubydb/indexes/index_manager.rb +408 -406
  296. data/lib/rubydb/indexes/index_scan.rb +466 -470
  297. data/lib/rubydb/migrations/migration.rb +253 -254
  298. data/lib/rubydb/migrations/migration_lock.rb +146 -146
  299. data/lib/rubydb/migrations/migration_manager.rb +187 -176
  300. data/lib/rubydb/migrations/migration_version.rb +71 -71
  301. data/lib/rubydb/migrations/schema_diff.rb +211 -211
  302. data/lib/rubydb/migrations/schema_version.rb +64 -64
  303. data/lib/rubydb/monitoring/events.rb +155 -160
  304. data/lib/rubydb/monitoring/health.rb +216 -222
  305. data/lib/rubydb/monitoring/logger.rb +188 -193
  306. data/lib/rubydb/monitoring/metrics.rb +363 -359
  307. data/lib/rubydb/monitoring/performance.rb +176 -176
  308. data/lib/rubydb/monitoring/statistics.rb +168 -170
  309. data/lib/rubydb/mvcc/garbage_collector.rb +199 -199
  310. data/lib/rubydb/mvcc/mvcc.rb +16 -16
  311. data/lib/rubydb/mvcc/snapshot.rb +146 -147
  312. data/lib/rubydb/mvcc/vacuum.rb +180 -180
  313. data/lib/rubydb/mvcc/version.rb +106 -106
  314. data/lib/rubydb/mvcc/version_store.rb +396 -398
  315. data/lib/rubydb/mvcc/visibility.rb +107 -109
  316. data/lib/rubydb/protocol/capabilities.rb +125 -125
  317. data/lib/rubydb/protocol/decoder.rb +142 -145
  318. data/lib/rubydb/protocol/encoder.rb +131 -136
  319. data/lib/rubydb/protocol/handshake.rb +306 -305
  320. data/lib/rubydb/protocol/message.rb +121 -121
  321. data/lib/rubydb/protocol/parameter_binder.rb +101 -0
  322. data/lib/rubydb/protocol/protocol.rb +276 -277
  323. data/lib/rubydb/protocol/version.rb +54 -54
  324. data/lib/rubydb/rails/adapter.rb +245 -239
  325. data/lib/rubydb/rails/connection.rb +312 -314
  326. data/lib/rubydb/rails/database_statements.rb +122 -122
  327. data/lib/rubydb/rails/migration.rb +131 -131
  328. data/lib/rubydb/rails/quoting.rb +109 -109
  329. data/lib/rubydb/rails/result.rb +117 -117
  330. data/lib/rubydb/rails/schema_statements.rb +339 -339
  331. data/lib/rubydb/rails/transaction.rb +105 -105
  332. data/lib/rubydb/rails/type.rb +126 -126
  333. data/lib/rubydb/recovery/checkpoint.rb +261 -257
  334. data/lib/rubydb/recovery/consistency.rb +457 -467
  335. data/lib/rubydb/recovery/corruption_detector.rb +5 -5
  336. data/lib/rubydb/recovery/crash_recovery.rb +381 -387
  337. data/lib/rubydb/recovery/recovery_manager.rb +204 -206
  338. data/lib/rubydb/recovery/redo.rb +235 -237
  339. data/lib/rubydb/recovery/undo.rb +204 -206
  340. data/lib/rubydb/replication/failover.rb +5 -5
  341. data/lib/rubydb/replication/fencing.rb +63 -63
  342. data/lib/rubydb/replication/primary.rb +461 -450
  343. data/lib/rubydb/replication/replica.rb +382 -384
  344. data/lib/rubydb/replication/replication_log.rb +194 -200
  345. data/lib/rubydb/replication/replication_manager.rb +307 -308
  346. data/lib/rubydb/replication/replication_slot.rb +293 -295
  347. data/lib/rubydb/replication/replication_stream.rb +198 -201
  348. data/lib/rubydb/rubydb.rb +570 -560
  349. data/lib/rubydb/security/access_control.rb +252 -254
  350. data/lib/rubydb/security/audit_log.rb +209 -213
  351. data/lib/rubydb/security/authentication.rb +302 -302
  352. data/lib/rubydb/security/authorization.rb +282 -282
  353. data/lib/rubydb/security/credentials.rb +192 -196
  354. data/lib/rubydb/security/password.rb +205 -215
  355. data/lib/rubydb/security/permissions.rb +74 -74
  356. data/lib/rubydb/security/role.rb +99 -101
  357. data/lib/rubydb/security/user.rb +86 -86
  358. data/lib/rubydb/server/connection.rb +383 -366
  359. data/lib/rubydb/server/connection_pool.rb +193 -193
  360. data/lib/rubydb/server/lifecycle.rb +227 -228
  361. data/lib/rubydb/server/listener.rb +139 -136
  362. data/lib/rubydb/server/request_handler.rb +277 -276
  363. data/lib/rubydb/server/server.rb +363 -364
  364. data/lib/rubydb/server/session.rb +416 -369
  365. data/lib/rubydb/server/worker.rb +206 -210
  366. data/lib/rubydb/server/worker_pool.rb +168 -168
  367. data/lib/rubydb/sql/ast/alter_table.rb +169 -169
  368. data/lib/rubydb/sql/ast/begin_transaction.rb +47 -47
  369. data/lib/rubydb/sql/ast/commit.rb +37 -37
  370. data/lib/rubydb/sql/ast/constraint.rb +92 -83
  371. data/lib/rubydb/sql/ast/create_database.rb +41 -41
  372. data/lib/rubydb/sql/ast/create_index.rb +61 -61
  373. data/lib/rubydb/sql/ast/create_schema.rb +52 -52
  374. data/lib/rubydb/sql/ast/create_table.rb +187 -187
  375. data/lib/rubydb/sql/ast/delete.rb +54 -54
  376. data/lib/rubydb/sql/ast/drop_database.rb +41 -41
  377. data/lib/rubydb/sql/ast/drop_index.rb +41 -41
  378. data/lib/rubydb/sql/ast/drop_schema.rb +49 -49
  379. data/lib/rubydb/sql/ast/drop_table.rb +49 -49
  380. data/lib/rubydb/sql/ast/explain.rb +64 -64
  381. data/lib/rubydb/sql/ast/expression.rb +617 -604
  382. data/lib/rubydb/sql/ast/insert.rb +66 -66
  383. data/lib/rubydb/sql/ast/node.rb +42 -42
  384. data/lib/rubydb/sql/ast/rollback.rb +63 -63
  385. data/lib/rubydb/sql/ast/savepoint.rb +59 -59
  386. data/lib/rubydb/sql/ast/select.rb +88 -88
  387. data/lib/rubydb/sql/ast/set_operation.rb +22 -20
  388. data/lib/rubydb/sql/ast/trigger.rb +35 -29
  389. data/lib/rubydb/sql/ast/update.rb +88 -88
  390. data/lib/rubydb/sql/ast/vacuum.rb +19 -19
  391. data/lib/rubydb/sql/ast/view.rb +38 -32
  392. data/lib/rubydb/sql/ast/with.rb +32 -32
  393. data/lib/rubydb/sql/grammar.rb +86 -86
  394. data/lib/rubydb/sql/keywords.rb +156 -156
  395. data/lib/rubydb/sql/lexer.rb +209 -214
  396. data/lib/rubydb/sql/operators.rb +100 -100
  397. data/lib/rubydb/sql/parser.rb +1167 -1170
  398. data/lib/rubydb/sql/planner/analyzer.rb +283 -302
  399. data/lib/rubydb/sql/planner/binder.rb +537 -550
  400. data/lib/rubydb/sql/planner/type_checker.rb +427 -431
  401. data/lib/rubydb/sql/token.rb +210 -210
  402. data/lib/rubydb/storage/buffer_frame.rb +44 -44
  403. data/lib/rubydb/storage/buffer_pool.rb +155 -155
  404. data/lib/rubydb/storage/database_lock.rb +74 -74
  405. data/lib/rubydb/storage/deserializer.rb +332 -342
  406. data/lib/rubydb/storage/engine.rb +2409 -2330
  407. data/lib/rubydb/storage/file_manager.rb +191 -187
  408. data/lib/rubydb/storage/free_space_map.rb +79 -81
  409. data/lib/rubydb/storage/page.rb +92 -94
  410. data/lib/rubydb/storage/page_allocator.rb +852 -855
  411. data/lib/rubydb/storage/page_header.rb +63 -67
  412. data/lib/rubydb/storage/page_manager.rb +127 -131
  413. data/lib/rubydb/storage/record.rb +58 -58
  414. data/lib/rubydb/storage/row.rb +78 -78
  415. data/lib/rubydb/storage/serializer.rb +51 -51
  416. data/lib/rubydb/storage/snapshot_reader.rb +167 -0
  417. data/lib/rubydb/storage/storage_layout.rb +151 -151
  418. data/lib/rubydb/storage/storage_manager.rb +114 -114
  419. data/lib/rubydb/storage/tuple.rb +458 -461
  420. data/lib/rubydb/storage/visibility_map.rb +964 -973
  421. data/lib/rubydb/transactions/commit_manager.rb +219 -220
  422. data/lib/rubydb/transactions/isolation.rb +98 -98
  423. data/lib/rubydb/transactions/lock.rb +76 -76
  424. data/lib/rubydb/transactions/lock_manager.rb +359 -362
  425. data/lib/rubydb/transactions/savepoint.rb +142 -143
  426. data/lib/rubydb/transactions/transaction.rb +214 -215
  427. data/lib/rubydb/transactions/transaction_id.rb +84 -84
  428. data/lib/rubydb/transactions/transaction_log.rb +256 -257
  429. data/lib/rubydb/transactions/transaction_manager.rb +434 -435
  430. data/lib/rubydb/types/bigint.rb +36 -36
  431. data/lib/rubydb/types/blob.rb +37 -37
  432. data/lib/rubydb/types/boolean.rb +34 -34
  433. data/lib/rubydb/types/date.rb +39 -39
  434. data/lib/rubydb/types/decimal.rb +48 -48
  435. data/lib/rubydb/types/float.rb +34 -34
  436. data/lib/rubydb/types/integer.rb +36 -36
  437. data/lib/rubydb/types/json.rb +41 -41
  438. data/lib/rubydb/types/null.rb +34 -34
  439. data/lib/rubydb/types/smallint.rb +36 -36
  440. data/lib/rubydb/types/text.rb +37 -37
  441. data/lib/rubydb/types/time.rb +46 -46
  442. data/lib/rubydb/types/timestamp.rb +39 -39
  443. data/lib/rubydb/types/type.rb +119 -119
  444. data/lib/rubydb/types/uuid.rb +47 -47
  445. data/lib/rubydb/types/varchar.rb +37 -37
  446. data/lib/rubydb/version.rb +32 -32
  447. data/lib/rubydb/wal/archive.rb +207 -193
  448. data/lib/rubydb/wal/checkpoint.rb +181 -183
  449. data/lib/rubydb/wal/lsn.rb +94 -94
  450. data/lib/rubydb/wal/reader.rb +259 -260
  451. data/lib/rubydb/wal/record.rb +105 -105
  452. data/lib/rubydb/wal/segment.rb +193 -193
  453. data/lib/rubydb/wal/wal.rb +481 -452
  454. data/lib/rubydb/wal/writer.rb +236 -236
  455. data/lib/rubydb.rb +7 -7
  456. data/packaging/docker/docker-compose.failover.yml +43 -43
  457. data/packaging/homebrew/rubydb.rb +19 -19
  458. data/rubydb.gemspec +70 -57
  459. data/scripts/benchmark +7 -7
  460. data/scripts/build_accelerator +49 -0
  461. data/scripts/durability_drill +37 -37
  462. data/scripts/fuzz +63 -63
  463. data/scripts/release +77 -42
  464. data/scripts/release_check +43 -43
  465. data/scripts/replication_failover_drill +268 -250
  466. data/scripts/replication_network_failover_drill +287 -255
  467. data/scripts/restore_drill +45 -45
  468. data/scripts/security +45 -0
  469. metadata +102 -1
@@ -1,308 +1,307 @@
1
- # frozen_string_literal: true
2
-
3
- require "thread"
4
- require "time"
5
- require "monitor"
6
-
7
- module RubyDB
8
- module Replication
9
- # ReplicationManager - Manages replication setup and coordination
10
- class ReplicationManager
11
- attr_reader :config, :primary, :replica, :stats, :mode
12
-
13
- # Replication modes
14
- MODE_OFF = :off
15
- MODE_PRIMARY = :primary
16
- MODE_REPLICA = :replica
17
- MODE_STANDBY = :standby
18
-
19
- def initialize(engine, config = {})
20
- @engine = engine
21
- @config = config
22
- @mode = config[:mode] || MODE_OFF
23
- @cluster_name = config[:cluster_name] || "rubydb_cluster"
24
- @node_id = config[:node_id] || generate_node_id
25
- @primary_host = config[:primary_host]
26
- @primary_port = config[:primary_port] || 7433
27
- @replication_port = config[:replication_port] || 7434
28
-
29
- @primary = nil
30
- @replica = nil
31
- @replication_slots = {}
32
- @health_checks = []
33
- @stats = {
34
- mode: @mode,
35
- node_id: @node_id,
36
- cluster_name: @cluster_name,
37
- healthy: true,
38
- last_health_check: nil,
39
- failovers: 0,
40
- switches: 0,
41
- errors: 0
42
- }
43
- @lock = Monitor.new
44
- @running = false
45
- @health_thread = nil
46
- @health_check_interval = config[:health_check_interval] || 10
47
-
48
- # Initialize based on mode
49
- initialize_mode
50
- end
51
-
52
- def start
53
- @lock.synchronize do
54
- return if @running
55
-
56
- @running = true
57
-
58
- case @mode
59
- when MODE_PRIMARY
60
- @primary&.start
61
- when MODE_REPLICA
62
- @replica&.start
63
- end
64
-
65
- # Start health check
66
- start_health_check
67
-
68
- puts "Replication manager started in #{@mode} mode"
69
- true
70
- end
71
- end
72
-
73
- def stop
74
- @lock.synchronize do
75
- return unless @running
76
-
77
- @running = false
78
-
79
- @health_thread&.kill
80
- @health_thread = nil
81
-
82
- @primary&.stop
83
- @replica&.stop
84
-
85
- puts "Replication manager stopped"
86
- true
87
- end
88
- end
89
-
90
- def switch_mode(new_mode, config = {})
91
- @lock.synchronize do
92
- old_mode = @mode
93
-
94
- # Stop current mode
95
- stop
96
-
97
- # Update config
98
- @mode = new_mode
99
- @config = @config.merge(config)
100
-
101
- # Initialize new mode
102
- initialize_mode
103
-
104
- # Start new mode
105
- start
106
-
107
- @stats[:switches] += 1
108
-
109
- { old_mode: old_mode, new_mode: new_mode }
110
- end
111
- end
112
-
113
- def promote_to_primary(recovery_point: nil)
114
- @lock.synchronize do
115
- if @mode != MODE_REPLICA
116
- return { success: false, error: "Not in replica mode" }
117
- end
118
-
119
- status = @replica&.replication_status
120
- promotable_state = status && [
121
- Replica::STATE_STREAMING,
122
- Replica::STATE_SYNCED,
123
- Replica::STATE_DISCONNECTED,
124
- Replica::STATE_CONNECTING
125
- ].include?(status[:state])
126
- caught_up_after_disconnect = status &&
127
- status[:last_replayed_lsn] &&
128
- status[:last_received_lsn] == status[:last_replayed_lsn]
129
- unless promotable_state && (status[:state] == Replica::STATE_STREAMING ||
130
- status[:state] == Replica::STATE_SYNCED ||
131
- caught_up_after_disconnect)
132
- return { success: false, error: "Replica is not synchronized enough for manual promotion" }
133
- end
134
-
135
- if status[:last_received_lsn] != status[:last_replayed_lsn]
136
- return {
137
- success: false,
138
- error: "Replica has unapplied replication data; promotion requires a caught-up recovery point"
139
- }
140
- end
141
-
142
- if recovery_point && status[:last_replayed_lsn].to_i < recovery_point.to_i
143
- return { success: false, error: "Replica has not reached recovery point #{recovery_point}" }
144
- end
145
-
146
- # Promote replica to primary. Automatic promotion is intentionally
147
- # not attempted; callers must explicitly invoke this operation.
148
- begin
149
- @replica&.promote_to_primary(recovery_point: recovery_point)
150
- rescue ReplicationError => e
151
- return { success: false, error: e.message }
152
- end
153
-
154
- # Switch mode
155
- @mode = MODE_PRIMARY
156
-
157
- # Create new primary
158
- @primary = Primary.new(@engine, @config)
159
- @primary.start
160
-
161
- @replica = nil
162
- @stats[:failovers] += 1
163
-
164
- { success: true, message: "Promoted to primary" }
165
- end
166
- end
167
-
168
- def demote_to_replica(primary_host, primary_port = nil)
169
- @lock.synchronize do
170
- if @mode != MODE_PRIMARY
171
- return { success: false, error: "Not in primary mode" }
172
- end
173
-
174
- # Stop primary
175
- @primary&.stop
176
-
177
- # Update config
178
- @config[:primary_host] = primary_host
179
- @config[:primary_port] = primary_port if primary_port
180
- @mode = MODE_REPLICA
181
-
182
- # Create new replica
183
- @replica = Replica.new(@engine, @config)
184
- @replica.start
185
-
186
- @primary = nil
187
- @stats[:switches] += 1
188
-
189
- { success: true, message: "Demoted to replica" }
190
- end
191
- end
192
-
193
- def health_check
194
- @lock.synchronize do
195
- @stats[:last_health_check] = Time.now
196
-
197
- result = case @mode
198
- when MODE_PRIMARY
199
- check_primary_health
200
- when MODE_REPLICA
201
- check_replica_health
202
- else
203
- { healthy: true, message: "Not in active replication mode" }
204
- end
205
-
206
- @stats[:healthy] = result[:healthy]
207
- @health_checks << result
208
-
209
- # Trim health checks
210
- if @health_checks.size > 100
211
- @health_checks.shift
212
- end
213
-
214
- result
215
- end
216
- end
217
-
218
- def get_replication_status
219
- @lock.synchronize do
220
- status = {
221
- mode: @mode,
222
- node_id: @node_id,
223
- cluster_name: @cluster_name,
224
- running: @running,
225
- healthy: @stats[:healthy],
226
- last_health_check: @stats[:last_health_check]&.iso8601
227
- }
228
-
229
- case @mode
230
- when MODE_PRIMARY
231
- status[:primary] = @primary&.get_replication_status
232
- when MODE_REPLICA
233
- status[:replica] = @replica&.replication_status
234
- end
235
-
236
- status
237
- end
238
- end
239
-
240
- def stats
241
- @lock.synchronize do
242
- @stats.merge({
243
- running: @running,
244
- health_checks: @health_checks.size,
245
- mode: @mode,
246
- node_id: @node_id
247
- })
248
- end
249
- end
250
-
251
- private
252
-
253
- def generate_node_id
254
- "node_#{Time.now.to_i}_#{rand(10000)}"
255
- end
256
-
257
- def initialize_mode
258
- case @mode
259
- when MODE_PRIMARY
260
- @primary = Primary.new(@engine, @config)
261
- when MODE_REPLICA
262
- unless @config[:primary_host]
263
- raise ConfigError, "Primary host required for replica mode"
264
- end
265
- @replica = Replica.new(@engine, @config)
266
- when MODE_OFF, MODE_STANDBY
267
- # No replication
268
- else
269
- @mode = MODE_OFF
270
- end
271
- end
272
-
273
- def start_health_check
274
- @health_thread = Thread.new do
275
- while @running
276
- sleep(@health_check_interval)
277
- begin
278
- health_check
279
- rescue => e
280
- @stats[:errors] += 1
281
- end
282
- end
283
- end
284
- end
285
-
286
- def check_primary_health
287
- if @primary&.running?
288
- { healthy: true, message: "Primary is running" }
289
- else
290
- { healthy: false, message: "Primary is not running" }
291
- end
292
- end
293
-
294
- def check_replica_health
295
- if @replica&.running?
296
- status = @replica.replication_status
297
- if status[:state] == :failed
298
- { healthy: false, message: "Replica in failed state" }
299
- else
300
- { healthy: true, message: "Replica is running" }
301
- end
302
- else
303
- { healthy: false, message: "Replica is not running" }
304
- end
305
- end
306
- end
307
- end
308
- end
1
+ # frozen_string_literal: true
2
+
3
+ require "time"
4
+ require "monitor"
5
+
6
+ module RubyDB
7
+ module Replication
8
+ # ReplicationManager - Manages replication setup and coordination
9
+ class ReplicationManager
10
+ attr_reader :config, :primary, :replica, :stats, :mode
11
+
12
+ # Replication modes
13
+ MODE_OFF = :off
14
+ MODE_PRIMARY = :primary
15
+ MODE_REPLICA = :replica
16
+ MODE_STANDBY = :standby
17
+
18
+ def initialize(engine, config = {})
19
+ @engine = engine
20
+ @config = config
21
+ @mode = config[:mode] || MODE_OFF
22
+ @cluster_name = config[:cluster_name] || "rubydb_cluster"
23
+ @node_id = config[:node_id] || generate_node_id
24
+ @primary_host = config[:primary_host]
25
+ @primary_port = config[:primary_port] || 7433
26
+ @replication_port = config[:replication_port] || 7434
27
+
28
+ @primary = nil
29
+ @replica = nil
30
+ @replication_slots = {}
31
+ @health_checks = []
32
+ @stats = {
33
+ mode: @mode,
34
+ node_id: @node_id,
35
+ cluster_name: @cluster_name,
36
+ healthy: true,
37
+ last_health_check: nil,
38
+ failovers: 0,
39
+ switches: 0,
40
+ errors: 0
41
+ }
42
+ @lock = Monitor.new
43
+ @running = false
44
+ @health_thread = nil
45
+ @health_check_interval = config[:health_check_interval] || 10
46
+
47
+ # Initialize based on mode
48
+ initialize_mode
49
+ end
50
+
51
+ def start
52
+ @lock.synchronize do
53
+ return if @running
54
+
55
+ @running = true
56
+
57
+ case @mode
58
+ when MODE_PRIMARY
59
+ @primary&.start
60
+ when MODE_REPLICA
61
+ @replica&.start
62
+ end
63
+
64
+ # Start health check
65
+ start_health_check
66
+
67
+ puts "Replication manager started in #{@mode} mode"
68
+ true
69
+ end
70
+ end
71
+
72
+ def stop
73
+ @lock.synchronize do
74
+ return unless @running
75
+
76
+ @running = false
77
+
78
+ @health_thread&.kill
79
+ @health_thread = nil
80
+
81
+ @primary&.stop
82
+ @replica&.stop
83
+
84
+ puts "Replication manager stopped"
85
+ true
86
+ end
87
+ end
88
+
89
+ def switch_mode(new_mode, config = {})
90
+ @lock.synchronize do
91
+ old_mode = @mode
92
+
93
+ # Stop current mode
94
+ stop
95
+
96
+ # Update config
97
+ @mode = new_mode
98
+ @config = @config.merge(config)
99
+
100
+ # Initialize new mode
101
+ initialize_mode
102
+
103
+ # Start new mode
104
+ start
105
+
106
+ @stats[:switches] += 1
107
+
108
+ {old_mode: old_mode, new_mode: new_mode}
109
+ end
110
+ end
111
+
112
+ def promote_to_primary(recovery_point: nil)
113
+ @lock.synchronize do
114
+ if @mode != MODE_REPLICA
115
+ return {success: false, error: "Not in replica mode"}
116
+ end
117
+
118
+ status = @replica&.replication_status
119
+ promotable_state = status && [
120
+ Replica::STATE_STREAMING,
121
+ Replica::STATE_SYNCED,
122
+ Replica::STATE_DISCONNECTED,
123
+ Replica::STATE_CONNECTING
124
+ ].include?(status[:state])
125
+ caught_up_after_disconnect = status &&
126
+ status[:last_replayed_lsn] &&
127
+ status[:last_received_lsn] == status[:last_replayed_lsn]
128
+ synchronized_state = status && [Replica::STATE_STREAMING, Replica::STATE_SYNCED].include?(status[:state]) ||
129
+ caught_up_after_disconnect
130
+ unless promotable_state && synchronized_state
131
+ return {success: false, error: "Replica is not synchronized enough for manual promotion"}
132
+ end
133
+
134
+ if status[:last_received_lsn] != status[:last_replayed_lsn]
135
+ return {
136
+ success: false,
137
+ error: "Replica has unapplied replication data; promotion requires a caught-up recovery point"
138
+ }
139
+ end
140
+
141
+ if recovery_point && status[:last_replayed_lsn].to_i < recovery_point.to_i
142
+ return {success: false, error: "Replica has not reached recovery point #{recovery_point}"}
143
+ end
144
+
145
+ # Promote replica to primary. Automatic promotion is intentionally
146
+ # not attempted; callers must explicitly invoke this operation.
147
+ begin
148
+ @replica&.promote_to_primary(recovery_point: recovery_point)
149
+ rescue ReplicationError => e
150
+ return {success: false, error: e.message}
151
+ end
152
+
153
+ # Switch mode
154
+ @mode = MODE_PRIMARY
155
+
156
+ # Create new primary
157
+ @primary = Primary.new(@engine, @config)
158
+ @primary.start
159
+
160
+ @replica = nil
161
+ @stats[:failovers] += 1
162
+
163
+ {success: true, message: "Promoted to primary"}
164
+ end
165
+ end
166
+
167
+ def demote_to_replica(primary_host, primary_port = nil)
168
+ @lock.synchronize do
169
+ if @mode != MODE_PRIMARY
170
+ return {success: false, error: "Not in primary mode"}
171
+ end
172
+
173
+ # Stop primary
174
+ @primary&.stop
175
+
176
+ # Update config
177
+ @config[:primary_host] = primary_host
178
+ @config[:primary_port] = primary_port if primary_port
179
+ @mode = MODE_REPLICA
180
+
181
+ # Create new replica
182
+ @replica = Replica.new(@engine, @config)
183
+ @replica.start
184
+
185
+ @primary = nil
186
+ @stats[:switches] += 1
187
+
188
+ {success: true, message: "Demoted to replica"}
189
+ end
190
+ end
191
+
192
+ def health_check
193
+ @lock.synchronize do
194
+ @stats[:last_health_check] = Time.now
195
+
196
+ result = case @mode
197
+ when MODE_PRIMARY
198
+ check_primary_health
199
+ when MODE_REPLICA
200
+ check_replica_health
201
+ else
202
+ {healthy: true, message: "Not in active replication mode"}
203
+ end
204
+
205
+ @stats[:healthy] = result[:healthy]
206
+ @health_checks << result
207
+
208
+ # Trim health checks
209
+ if @health_checks.size > 100
210
+ @health_checks.shift
211
+ end
212
+
213
+ result
214
+ end
215
+ end
216
+
217
+ def get_replication_status
218
+ @lock.synchronize do
219
+ status = {
220
+ mode: @mode,
221
+ node_id: @node_id,
222
+ cluster_name: @cluster_name,
223
+ running: @running,
224
+ healthy: @stats[:healthy],
225
+ last_health_check: @stats[:last_health_check]&.iso8601
226
+ }
227
+
228
+ case @mode
229
+ when MODE_PRIMARY
230
+ status[:primary] = @primary&.get_replication_status
231
+ when MODE_REPLICA
232
+ status[:replica] = @replica&.replication_status
233
+ end
234
+
235
+ status
236
+ end
237
+ end
238
+
239
+ def stats
240
+ @lock.synchronize do
241
+ @stats.merge({
242
+ running: @running,
243
+ health_checks: @health_checks.size,
244
+ mode: @mode,
245
+ node_id: @node_id
246
+ })
247
+ end
248
+ end
249
+
250
+ private
251
+
252
+ def generate_node_id
253
+ "node_#{Time.now.to_i}_#{rand(10000)}"
254
+ end
255
+
256
+ def initialize_mode
257
+ case @mode
258
+ when MODE_PRIMARY
259
+ @primary = Primary.new(@engine, @config)
260
+ when MODE_REPLICA
261
+ unless @config[:primary_host]
262
+ raise ConfigError, "Primary host required for replica mode"
263
+ end
264
+ @replica = Replica.new(@engine, @config)
265
+ when MODE_OFF, MODE_STANDBY
266
+ # No replication
267
+ else
268
+ @mode = MODE_OFF
269
+ end
270
+ end
271
+
272
+ def start_health_check
273
+ @health_thread = Thread.new do
274
+ while @running
275
+ sleep(@health_check_interval)
276
+ begin
277
+ health_check
278
+ rescue
279
+ @stats[:errors] += 1
280
+ end
281
+ end
282
+ end
283
+ end
284
+
285
+ def check_primary_health
286
+ if @primary&.running?
287
+ {healthy: true, message: "Primary is running"}
288
+ else
289
+ {healthy: false, message: "Primary is not running"}
290
+ end
291
+ end
292
+
293
+ def check_replica_health
294
+ if @replica&.running?
295
+ status = @replica.replication_status
296
+ if status[:state] == :failed
297
+ {healthy: false, message: "Replica in failed state"}
298
+ else
299
+ {healthy: true, message: "Replica is running"}
300
+ end
301
+ else
302
+ {healthy: false, message: "Replica is not running"}
303
+ end
304
+ end
305
+ end
306
+ end
307
+ end