rubydb 0.1.4 → 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (469) hide show
  1. checksums.yaml +4 -4
  2. data/.github/PULL_REQUEST_TEMPLATE.md +15 -15
  3. data/.github/workflows/benchmark.yml +26 -26
  4. data/.github/workflows/compatibility.yml +63 -63
  5. data/.github/workflows/fuzz.yml +33 -33
  6. data/.github/workflows/lint.yml +21 -21
  7. data/.github/workflows/operations.yml +24 -24
  8. data/.github/workflows/production-validation.yml +111 -111
  9. data/.github/workflows/release.yml +77 -77
  10. data/.github/workflows/security.yml +39 -37
  11. data/.github/workflows/test.yml +26 -26
  12. data/.github/workflows/workload.yml +58 -58
  13. data/.gitignore +16 -5
  14. data/.rubocop.yml +50 -44
  15. data/.standard.yml +9 -14
  16. data/ARCHITECTURE.md +21 -21
  17. data/CHANGELOG.md +57 -27
  18. data/CODE_OF_CONDUCT.md +13 -13
  19. data/CONTRIBUTING.md +29 -29
  20. data/GOVERNANCE.md +16 -16
  21. data/Gemfile +18 -17
  22. data/Gemfile.lock +125 -71
  23. data/README.md +168 -12
  24. data/ROADMAP.md +27 -27
  25. data/Rakefile +76 -71
  26. data/SECURITY.md +54 -54
  27. data/SUPPORT.md +14 -14
  28. data/accelerator/bin/SHA256SUMS +6 -0
  29. data/accelerator/bin/rubydb-accelerator-darwin-amd64 +0 -0
  30. data/accelerator/bin/rubydb-accelerator-darwin-arm64 +0 -0
  31. data/accelerator/bin/rubydb-accelerator-linux-amd64 +0 -0
  32. data/accelerator/bin/rubydb-accelerator-linux-arm64 +0 -0
  33. data/accelerator/bin/rubydb-accelerator-windows-amd64.exe +0 -0
  34. data/accelerator/bin/rubydb-accelerator-windows-arm64.exe +0 -0
  35. data/accelerator/cmd/rubydb-accelerator/main.go +11 -0
  36. data/accelerator/go.mod +3 -0
  37. data/accelerator/internal/execution/aggregate.go +94 -0
  38. data/accelerator/internal/execution/distinct.go +22 -0
  39. data/accelerator/internal/execution/filter.go +73 -0
  40. data/accelerator/internal/execution/join.go +79 -0
  41. data/accelerator/internal/execution/operators.go +167 -0
  42. data/accelerator/internal/execution/scan.go +20 -0
  43. data/accelerator/internal/execution/sort.go +62 -0
  44. data/accelerator/internal/execution/types.go +136 -0
  45. data/accelerator/internal/execution/value.go +67 -0
  46. data/accelerator/internal/memory/arena.go +47 -0
  47. data/accelerator/internal/memory/reuse.go +22 -0
  48. data/accelerator/internal/metrics/registry.go +67 -0
  49. data/accelerator/internal/parallel/bounded_queue.go +56 -0
  50. data/accelerator/internal/parallel/scheduler.go +47 -0
  51. data/accelerator/internal/parallel/worker_pool.go +53 -0
  52. data/accelerator/internal/protocol/cancellation.go +48 -0
  53. data/accelerator/internal/protocol/columnar.go +263 -0
  54. data/accelerator/internal/protocol/frame.go +187 -0
  55. data/accelerator/internal/runtime/worker.go +521 -0
  56. data/accelerator/internal/storage/page_reader.go +81 -0
  57. data/accelerator/internal/storage/snapshot_scan.go +539 -0
  58. data/accelerator/internal/wal/checksum.go +13 -0
  59. data/accelerator/internal/wal/compression.go +41 -0
  60. data/accelerator/internal/wal/group_commit.go +24 -0
  61. data/accelerator/internal/wal/record_encoder.go +40 -0
  62. data/adapters/activerecord/Gemfile +11 -11
  63. data/adapters/activerecord/README.md +8 -3
  64. data/adapters/activerecord/lib/active_record/connection_adapters/rubydb_adapter.rb +881 -879
  65. data/adapters/activerecord/rubydb-activerecord.gemspec +21 -21
  66. data/adapters/activerecord/spec/rubydb_adapter_integration_spec.rb +143 -143
  67. data/adapters/ruby/README.md +18 -18
  68. data/adapters/sequel/README.md +11 -11
  69. data/config/monitoring/prometheus-alerts.yml +39 -39
  70. data/config/production.yml +36 -36
  71. data/docs/README.md +77 -71
  72. data/docs/architecture/concurrency.md +14 -14
  73. data/docs/architecture/current-state.md +125 -125
  74. data/docs/architecture/execution-engine.md +25 -25
  75. data/docs/architecture/go-accelerator.md +179 -0
  76. data/docs/architecture/indexes.md +19 -19
  77. data/docs/architecture/mvcc.md +19 -19
  78. data/docs/architecture/overview.md +13 -13
  79. data/docs/architecture/pages.md +11 -11
  80. data/docs/architecture/production-roadmap.md +82 -82
  81. data/docs/architecture/query-planner.md +20 -20
  82. data/docs/architecture/recovery.md +18 -18
  83. data/docs/architecture/sql-engine.md +12 -12
  84. data/docs/architecture/storage-engine.md +14 -14
  85. data/docs/architecture/transactions.md +10 -10
  86. data/docs/architecture/wal.md +28 -28
  87. data/docs/cli-cheatsheet.md +98 -98
  88. data/docs/cli.md +299 -275
  89. data/docs/contributing/architecture.md +9 -9
  90. data/docs/contributing/benchmarking.md +30 -14
  91. data/docs/contributing/development.md +16 -16
  92. data/docs/contributing/release-process.md +49 -49
  93. data/docs/contributing/testing.md +16 -16
  94. data/docs/debugging.md +229 -229
  95. data/docs/developer/branching.md +10 -10
  96. data/docs/developer/database-diff.md +10 -10
  97. data/docs/developer/local-development.md +49 -17
  98. data/docs/developer/snapshots.md +9 -9
  99. data/docs/developer/temporal-data.md +10 -10
  100. data/docs/developer-guide.md +297 -297
  101. data/docs/getting-started/first-database.md +16 -16
  102. data/docs/getting-started/first-query.md +13 -13
  103. data/docs/getting-started/installation.md +19 -19
  104. data/docs/getting-started/local-to-production.md +300 -300
  105. data/docs/getting-started/quickstart.md +17 -17
  106. data/docs/getting-started/rails.md +16 -16
  107. data/docs/hardening_backlog.md +93 -93
  108. data/docs/lessons-learned.md +112 -112
  109. data/docs/operations/backups.md +33 -33
  110. data/docs/operations/disaster-recovery.md +31 -31
  111. data/docs/operations/failover.md +30 -30
  112. data/docs/operations/monitoring.md +25 -25
  113. data/docs/operations/production-guide.md +295 -295
  114. data/docs/operations/production-runbook.md +45 -45
  115. data/docs/operations/replication.md +33 -33
  116. data/docs/operations/restore.md +6 -6
  117. data/docs/operations/runbook.md +34 -34
  118. data/docs/operations/upgrades.md +14 -14
  119. data/docs/operations/workload-testing.md +17 -17
  120. data/docs/production-readiness.md +118 -118
  121. data/docs/production_validation.md +150 -150
  122. data/docs/rails/active-record.md +11 -11
  123. data/docs/rails/compatibility-guide.md +90 -90
  124. data/docs/rails/database-yml.md +92 -92
  125. data/docs/rails/installation.md +17 -17
  126. data/docs/rails/migrations.md +17 -17
  127. data/docs/rails/production.md +82 -82
  128. data/docs/rails/troubleshooting.md +18 -18
  129. data/docs/release.md +25 -25
  130. data/docs/server/architecture.md +10 -10
  131. data/docs/server/authentication.md +10 -10
  132. data/docs/server/configuration.md +16 -16
  133. data/docs/server/connection-pooling.md +10 -10
  134. data/docs/server/deployment.md +10 -10
  135. data/docs/server/protocol.md +12 -12
  136. data/docs/sql/compatibility-guide.md +82 -82
  137. data/docs/sql/compatibility.md +39 -39
  138. data/docs/sql/data-types.md +10 -10
  139. data/docs/sql/functions.md +9 -9
  140. data/docs/sql/joins.md +9 -9
  141. data/docs/sql/operators.md +9 -9
  142. data/docs/sql/sqlite-compatibility.md +21 -21
  143. data/docs/sql/syntax.md +10 -10
  144. data/docs/sql/transactions.md +10 -10
  145. data/docs/troubleshooting.md +244 -244
  146. data/lessons/01-foundations.md +73 -0
  147. data/lessons/02-local-development.md +121 -0
  148. data/lessons/03-embedded-rubydb.md +99 -0
  149. data/lessons/04-rails-complex-apps.md +138 -0
  150. data/lessons/05-rubydb-production-server.md +237 -0
  151. data/lessons/06-postgresql-massive-apps.md +96 -0
  152. data/lessons/07-hybrid-microservices.md +179 -0
  153. data/lessons/08-migrations-backups-recovery.md +86 -0
  154. data/lessons/09-observability-security-scale.md +87 -0
  155. data/lessons/10-release-readiness.md +192 -0
  156. data/lessons/11-community-adapter.md +323 -0
  157. data/lessons/12-rails-ecommerce-pressure.md +263 -0
  158. data/lib/rubydb/accelerator/client.rb +451 -0
  159. data/lib/rubydb/accelerator/error.rb +22 -0
  160. data/lib/rubydb/accelerator/manager.rb +606 -0
  161. data/lib/rubydb/accelerator.rb +13 -0
  162. data/lib/rubydb/backup/archive.rb +332 -334
  163. data/lib/rubydb/backup/backup.rb +400 -401
  164. data/lib/rubydb/backup/incremental.rb +349 -353
  165. data/lib/rubydb/backup/restore.rb +289 -290
  166. data/lib/rubydb/backup/snapshot.rb +265 -267
  167. data/lib/rubydb/backup/verification.rb +276 -279
  168. data/lib/rubydb/branching/branch.rb +181 -181
  169. data/lib/rubydb/branching/branch_manager.rb +307 -311
  170. data/lib/rubydb/branching/branch_metadata.rb +140 -140
  171. data/lib/rubydb/branching/checkout.rb +165 -166
  172. data/lib/rubydb/branching/copy_on_write.rb +272 -272
  173. data/lib/rubydb/branching/diff.rb +137 -138
  174. data/lib/rubydb/branching/merge.rb +282 -285
  175. data/lib/rubydb/build_info.rb +15 -15
  176. data/lib/rubydb/catalog/catalog.rb +391 -391
  177. data/lib/rubydb/catalog/column.rb +112 -112
  178. data/lib/rubydb/catalog/constraint.rb +180 -180
  179. data/lib/rubydb/catalog/database.rb +184 -184
  180. data/lib/rubydb/catalog/index.rb +97 -97
  181. data/lib/rubydb/catalog/schema.rb +103 -103
  182. data/lib/rubydb/catalog/sequence.rb +90 -90
  183. data/lib/rubydb/catalog/system_catalog.rb +698 -698
  184. data/lib/rubydb/catalog/table.rb +178 -178
  185. data/lib/rubydb/catalog/trigger.rb +102 -102
  186. data/lib/rubydb/catalog/view.rb +66 -66
  187. data/lib/rubydb/cli/application.rb +168 -163
  188. data/lib/rubydb/cli/commands/accelerator.rb +72 -0
  189. data/lib/rubydb/cli/commands/backup.rb +80 -81
  190. data/lib/rubydb/cli/commands/branch.rb +72 -72
  191. data/lib/rubydb/cli/commands/checkout.rb +54 -54
  192. data/lib/rubydb/cli/commands/create.rb +58 -58
  193. data/lib/rubydb/cli/commands/diff.rb +76 -77
  194. data/lib/rubydb/cli/commands/doctor.rb +77 -74
  195. data/lib/rubydb/cli/commands/drop.rb +57 -57
  196. data/lib/rubydb/cli/commands/init.rb +101 -102
  197. data/lib/rubydb/cli/commands/inspect.rb +95 -95
  198. data/lib/rubydb/cli/commands/merge.rb +63 -63
  199. data/lib/rubydb/cli/commands/migrate.rb +62 -62
  200. data/lib/rubydb/cli/commands/restart.rb +42 -39
  201. data/lib/rubydb/cli/commands/restore.rb +121 -121
  202. data/lib/rubydb/cli/commands/shell.rb +365 -365
  203. data/lib/rubydb/cli/commands/snapshot.rb +79 -79
  204. data/lib/rubydb/cli/commands/start.rb +88 -82
  205. data/lib/rubydb/cli/commands/status.rb +96 -92
  206. data/lib/rubydb/cli/commands/stop.rb +47 -47
  207. data/lib/rubydb/cli/commands/vacuum.rb +58 -58
  208. data/lib/rubydb/cli/formatter.rb +221 -221
  209. data/lib/rubydb/cli/output.rb +168 -168
  210. data/lib/rubydb/client/client.rb +309 -304
  211. data/lib/rubydb/client/connection.rb +429 -415
  212. data/lib/rubydb/client/connection_pool.rb +168 -168
  213. data/lib/rubydb/client/connection_url.rb +96 -96
  214. data/lib/rubydb/client/prepared_statement.rb +60 -60
  215. data/lib/rubydb/client/result.rb +127 -123
  216. data/lib/rubydb/client/statement.rb +52 -52
  217. data/lib/rubydb/client/transaction.rb +130 -130
  218. data/lib/rubydb/concurrency/concurrency.rb +19 -19
  219. data/lib/rubydb/concurrency/deadlock_detector.rb +148 -150
  220. data/lib/rubydb/concurrency/latch.rb +101 -101
  221. data/lib/rubydb/concurrency/lock_graph.rb +163 -165
  222. data/lib/rubydb/concurrency/mutex.rb +181 -183
  223. data/lib/rubydb/concurrency/rw_lock.rb +180 -180
  224. data/lib/rubydb/concurrency/scheduler.rb +248 -250
  225. data/lib/rubydb/concurrency/worker_pool.rb +145 -143
  226. data/lib/rubydb/configuration/config.rb +170 -170
  227. data/lib/rubydb/configuration/defaults.rb +191 -179
  228. data/lib/rubydb/configuration/environment.rb +152 -152
  229. data/lib/rubydb/configuration/parser.rb +185 -185
  230. data/lib/rubydb/configuration/validation.rb +228 -221
  231. data/lib/rubydb/constants.rb +74 -74
  232. data/lib/rubydb/constraints/check.rb +181 -181
  233. data/lib/rubydb/constraints/constraint.rb +101 -101
  234. data/lib/rubydb/constraints/foreign_key.rb +130 -130
  235. data/lib/rubydb/constraints/not_null.rb +64 -64
  236. data/lib/rubydb/constraints/primary_key.rb +99 -99
  237. data/lib/rubydb/constraints/unique.rb +106 -108
  238. data/lib/rubydb/constraints/validator.rb +349 -350
  239. data/lib/rubydb/errors/authentication_error.rb +10 -10
  240. data/lib/rubydb/errors/authorization_error.rb +23 -23
  241. data/lib/rubydb/errors/client_error.rb +10 -10
  242. data/lib/rubydb/errors/configuration_error.rb +10 -10
  243. data/lib/rubydb/errors/connection_error.rb +10 -10
  244. data/lib/rubydb/errors/constraint_error.rb +23 -23
  245. data/lib/rubydb/errors/corruption_error.rb +10 -10
  246. data/lib/rubydb/errors/database_error.rb +10 -10
  247. data/lib/rubydb/errors/error.rb +20 -20
  248. data/lib/rubydb/errors/execution_error.rb +10 -10
  249. data/lib/rubydb/errors/parser_error.rb +10 -10
  250. data/lib/rubydb/errors/recovery_error.rb +10 -10
  251. data/lib/rubydb/errors/replication_error.rb +10 -10
  252. data/lib/rubydb/errors/server_error.rb +6 -6
  253. data/lib/rubydb/errors/storage_error.rb +10 -10
  254. data/lib/rubydb/errors/transaction_error.rb +10 -10
  255. data/lib/rubydb/execution/accelerator_dispatch.rb +30 -0
  256. data/lib/rubydb/execution/aggregate_executor.rb +134 -138
  257. data/lib/rubydb/execution/cost_model.rb +72 -0
  258. data/lib/rubydb/execution/delete_executor.rb +110 -112
  259. data/lib/rubydb/execution/distinct_executor.rb +131 -135
  260. data/lib/rubydb/execution/executor.rb +1544 -1188
  261. data/lib/rubydb/execution/expression.rb +191 -193
  262. data/lib/rubydb/execution/index_scan.rb +142 -142
  263. data/lib/rubydb/execution/insert_executor.rb +215 -217
  264. data/lib/rubydb/execution/join_executor.rb +243 -249
  265. data/lib/rubydb/execution/limit_executor.rb +83 -85
  266. data/lib/rubydb/execution/operator_selection.rb +57 -0
  267. data/lib/rubydb/execution/optimizer.rb +227 -215
  268. data/lib/rubydb/execution/physical_plan.rb +47 -0
  269. data/lib/rubydb/execution/plan.rb +355 -353
  270. data/lib/rubydb/execution/planner.rb +508 -536
  271. data/lib/rubydb/execution/predicate.rb +235 -235
  272. data/lib/rubydb/execution/scan.rb +49 -49
  273. data/lib/rubydb/execution/sequential_scan.rb +63 -63
  274. data/lib/rubydb/execution/sort_executor.rb +194 -185
  275. data/lib/rubydb/execution/update_executor.rb +160 -162
  276. data/lib/rubydb/functions/aggregate.rb +70 -70
  277. data/lib/rubydb/functions/date_functions.rb +274 -278
  278. data/lib/rubydb/functions/function.rb +85 -85
  279. data/lib/rubydb/functions/json_functions.rb +231 -215
  280. data/lib/rubydb/functions/numeric_functions.rb +346 -346
  281. data/lib/rubydb/functions/scalar.rb +52 -52
  282. data/lib/rubydb/functions/string_functions.rb +383 -383
  283. data/lib/rubydb/functions/system_functions.rb +258 -246
  284. data/lib/rubydb/history/as_of.rb +238 -238
  285. data/lib/rubydb/history/change.rb +105 -105
  286. data/lib/rubydb/history/history.rb +131 -131
  287. data/lib/rubydb/history/history_manager.rb +228 -229
  288. data/lib/rubydb/history/temporal_query.rb +202 -202
  289. data/lib/rubydb/history/timeline.rb +144 -144
  290. data/lib/rubydb/indexes/btree.rb +215 -186
  291. data/lib/rubydb/indexes/btree_cursor.rb +258 -258
  292. data/lib/rubydb/indexes/btree_node.rb +384 -385
  293. data/lib/rubydb/indexes/hash_index.rb +150 -150
  294. data/lib/rubydb/indexes/index.rb +71 -71
  295. data/lib/rubydb/indexes/index_manager.rb +408 -406
  296. data/lib/rubydb/indexes/index_scan.rb +466 -470
  297. data/lib/rubydb/migrations/migration.rb +253 -254
  298. data/lib/rubydb/migrations/migration_lock.rb +146 -146
  299. data/lib/rubydb/migrations/migration_manager.rb +187 -176
  300. data/lib/rubydb/migrations/migration_version.rb +71 -71
  301. data/lib/rubydb/migrations/schema_diff.rb +211 -211
  302. data/lib/rubydb/migrations/schema_version.rb +64 -64
  303. data/lib/rubydb/monitoring/events.rb +155 -160
  304. data/lib/rubydb/monitoring/health.rb +216 -222
  305. data/lib/rubydb/monitoring/logger.rb +188 -193
  306. data/lib/rubydb/monitoring/metrics.rb +363 -359
  307. data/lib/rubydb/monitoring/performance.rb +176 -176
  308. data/lib/rubydb/monitoring/statistics.rb +168 -170
  309. data/lib/rubydb/mvcc/garbage_collector.rb +199 -199
  310. data/lib/rubydb/mvcc/mvcc.rb +16 -16
  311. data/lib/rubydb/mvcc/snapshot.rb +146 -147
  312. data/lib/rubydb/mvcc/vacuum.rb +180 -180
  313. data/lib/rubydb/mvcc/version.rb +106 -106
  314. data/lib/rubydb/mvcc/version_store.rb +396 -398
  315. data/lib/rubydb/mvcc/visibility.rb +107 -109
  316. data/lib/rubydb/protocol/capabilities.rb +125 -125
  317. data/lib/rubydb/protocol/decoder.rb +142 -145
  318. data/lib/rubydb/protocol/encoder.rb +131 -136
  319. data/lib/rubydb/protocol/handshake.rb +306 -305
  320. data/lib/rubydb/protocol/message.rb +121 -121
  321. data/lib/rubydb/protocol/parameter_binder.rb +101 -0
  322. data/lib/rubydb/protocol/protocol.rb +276 -277
  323. data/lib/rubydb/protocol/version.rb +54 -54
  324. data/lib/rubydb/rails/adapter.rb +245 -239
  325. data/lib/rubydb/rails/connection.rb +312 -314
  326. data/lib/rubydb/rails/database_statements.rb +122 -122
  327. data/lib/rubydb/rails/migration.rb +131 -131
  328. data/lib/rubydb/rails/quoting.rb +109 -109
  329. data/lib/rubydb/rails/result.rb +117 -117
  330. data/lib/rubydb/rails/schema_statements.rb +339 -339
  331. data/lib/rubydb/rails/transaction.rb +105 -105
  332. data/lib/rubydb/rails/type.rb +126 -126
  333. data/lib/rubydb/recovery/checkpoint.rb +261 -257
  334. data/lib/rubydb/recovery/consistency.rb +457 -467
  335. data/lib/rubydb/recovery/corruption_detector.rb +5 -5
  336. data/lib/rubydb/recovery/crash_recovery.rb +381 -387
  337. data/lib/rubydb/recovery/recovery_manager.rb +204 -206
  338. data/lib/rubydb/recovery/redo.rb +235 -237
  339. data/lib/rubydb/recovery/undo.rb +204 -206
  340. data/lib/rubydb/replication/failover.rb +5 -5
  341. data/lib/rubydb/replication/fencing.rb +63 -63
  342. data/lib/rubydb/replication/primary.rb +461 -450
  343. data/lib/rubydb/replication/replica.rb +382 -384
  344. data/lib/rubydb/replication/replication_log.rb +194 -200
  345. data/lib/rubydb/replication/replication_manager.rb +307 -308
  346. data/lib/rubydb/replication/replication_slot.rb +293 -295
  347. data/lib/rubydb/replication/replication_stream.rb +198 -201
  348. data/lib/rubydb/rubydb.rb +570 -560
  349. data/lib/rubydb/security/access_control.rb +252 -254
  350. data/lib/rubydb/security/audit_log.rb +209 -213
  351. data/lib/rubydb/security/authentication.rb +302 -302
  352. data/lib/rubydb/security/authorization.rb +282 -282
  353. data/lib/rubydb/security/credentials.rb +192 -196
  354. data/lib/rubydb/security/password.rb +205 -215
  355. data/lib/rubydb/security/permissions.rb +74 -74
  356. data/lib/rubydb/security/role.rb +99 -101
  357. data/lib/rubydb/security/user.rb +86 -86
  358. data/lib/rubydb/server/connection.rb +383 -366
  359. data/lib/rubydb/server/connection_pool.rb +193 -193
  360. data/lib/rubydb/server/lifecycle.rb +227 -228
  361. data/lib/rubydb/server/listener.rb +139 -136
  362. data/lib/rubydb/server/request_handler.rb +277 -276
  363. data/lib/rubydb/server/server.rb +363 -364
  364. data/lib/rubydb/server/session.rb +416 -369
  365. data/lib/rubydb/server/worker.rb +206 -210
  366. data/lib/rubydb/server/worker_pool.rb +168 -168
  367. data/lib/rubydb/sql/ast/alter_table.rb +169 -169
  368. data/lib/rubydb/sql/ast/begin_transaction.rb +47 -47
  369. data/lib/rubydb/sql/ast/commit.rb +37 -37
  370. data/lib/rubydb/sql/ast/constraint.rb +92 -83
  371. data/lib/rubydb/sql/ast/create_database.rb +41 -41
  372. data/lib/rubydb/sql/ast/create_index.rb +61 -61
  373. data/lib/rubydb/sql/ast/create_schema.rb +52 -52
  374. data/lib/rubydb/sql/ast/create_table.rb +187 -187
  375. data/lib/rubydb/sql/ast/delete.rb +54 -54
  376. data/lib/rubydb/sql/ast/drop_database.rb +41 -41
  377. data/lib/rubydb/sql/ast/drop_index.rb +41 -41
  378. data/lib/rubydb/sql/ast/drop_schema.rb +49 -49
  379. data/lib/rubydb/sql/ast/drop_table.rb +49 -49
  380. data/lib/rubydb/sql/ast/explain.rb +64 -64
  381. data/lib/rubydb/sql/ast/expression.rb +617 -604
  382. data/lib/rubydb/sql/ast/insert.rb +66 -66
  383. data/lib/rubydb/sql/ast/node.rb +42 -42
  384. data/lib/rubydb/sql/ast/rollback.rb +63 -63
  385. data/lib/rubydb/sql/ast/savepoint.rb +59 -59
  386. data/lib/rubydb/sql/ast/select.rb +88 -88
  387. data/lib/rubydb/sql/ast/set_operation.rb +22 -20
  388. data/lib/rubydb/sql/ast/trigger.rb +35 -29
  389. data/lib/rubydb/sql/ast/update.rb +88 -88
  390. data/lib/rubydb/sql/ast/vacuum.rb +19 -19
  391. data/lib/rubydb/sql/ast/view.rb +38 -32
  392. data/lib/rubydb/sql/ast/with.rb +32 -32
  393. data/lib/rubydb/sql/grammar.rb +86 -86
  394. data/lib/rubydb/sql/keywords.rb +156 -156
  395. data/lib/rubydb/sql/lexer.rb +209 -214
  396. data/lib/rubydb/sql/operators.rb +100 -100
  397. data/lib/rubydb/sql/parser.rb +1167 -1170
  398. data/lib/rubydb/sql/planner/analyzer.rb +283 -302
  399. data/lib/rubydb/sql/planner/binder.rb +537 -550
  400. data/lib/rubydb/sql/planner/type_checker.rb +427 -431
  401. data/lib/rubydb/sql/token.rb +210 -210
  402. data/lib/rubydb/storage/buffer_frame.rb +44 -44
  403. data/lib/rubydb/storage/buffer_pool.rb +155 -155
  404. data/lib/rubydb/storage/database_lock.rb +74 -74
  405. data/lib/rubydb/storage/deserializer.rb +332 -342
  406. data/lib/rubydb/storage/engine.rb +2409 -2330
  407. data/lib/rubydb/storage/file_manager.rb +191 -187
  408. data/lib/rubydb/storage/free_space_map.rb +79 -81
  409. data/lib/rubydb/storage/page.rb +92 -94
  410. data/lib/rubydb/storage/page_allocator.rb +852 -855
  411. data/lib/rubydb/storage/page_header.rb +63 -67
  412. data/lib/rubydb/storage/page_manager.rb +127 -131
  413. data/lib/rubydb/storage/record.rb +58 -58
  414. data/lib/rubydb/storage/row.rb +78 -78
  415. data/lib/rubydb/storage/serializer.rb +51 -51
  416. data/lib/rubydb/storage/snapshot_reader.rb +167 -0
  417. data/lib/rubydb/storage/storage_layout.rb +151 -151
  418. data/lib/rubydb/storage/storage_manager.rb +114 -114
  419. data/lib/rubydb/storage/tuple.rb +458 -461
  420. data/lib/rubydb/storage/visibility_map.rb +964 -973
  421. data/lib/rubydb/transactions/commit_manager.rb +219 -220
  422. data/lib/rubydb/transactions/isolation.rb +98 -98
  423. data/lib/rubydb/transactions/lock.rb +76 -76
  424. data/lib/rubydb/transactions/lock_manager.rb +359 -362
  425. data/lib/rubydb/transactions/savepoint.rb +142 -143
  426. data/lib/rubydb/transactions/transaction.rb +214 -215
  427. data/lib/rubydb/transactions/transaction_id.rb +84 -84
  428. data/lib/rubydb/transactions/transaction_log.rb +256 -257
  429. data/lib/rubydb/transactions/transaction_manager.rb +434 -435
  430. data/lib/rubydb/types/bigint.rb +36 -36
  431. data/lib/rubydb/types/blob.rb +37 -37
  432. data/lib/rubydb/types/boolean.rb +34 -34
  433. data/lib/rubydb/types/date.rb +39 -39
  434. data/lib/rubydb/types/decimal.rb +48 -48
  435. data/lib/rubydb/types/float.rb +34 -34
  436. data/lib/rubydb/types/integer.rb +36 -36
  437. data/lib/rubydb/types/json.rb +41 -41
  438. data/lib/rubydb/types/null.rb +34 -34
  439. data/lib/rubydb/types/smallint.rb +36 -36
  440. data/lib/rubydb/types/text.rb +37 -37
  441. data/lib/rubydb/types/time.rb +46 -46
  442. data/lib/rubydb/types/timestamp.rb +39 -39
  443. data/lib/rubydb/types/type.rb +119 -119
  444. data/lib/rubydb/types/uuid.rb +47 -47
  445. data/lib/rubydb/types/varchar.rb +37 -37
  446. data/lib/rubydb/version.rb +32 -32
  447. data/lib/rubydb/wal/archive.rb +207 -193
  448. data/lib/rubydb/wal/checkpoint.rb +181 -183
  449. data/lib/rubydb/wal/lsn.rb +94 -94
  450. data/lib/rubydb/wal/reader.rb +259 -260
  451. data/lib/rubydb/wal/record.rb +105 -105
  452. data/lib/rubydb/wal/segment.rb +193 -193
  453. data/lib/rubydb/wal/wal.rb +481 -452
  454. data/lib/rubydb/wal/writer.rb +236 -236
  455. data/lib/rubydb.rb +7 -7
  456. data/packaging/docker/docker-compose.failover.yml +43 -43
  457. data/packaging/homebrew/rubydb.rb +19 -19
  458. data/rubydb.gemspec +70 -57
  459. data/scripts/benchmark +7 -7
  460. data/scripts/build_accelerator +49 -0
  461. data/scripts/durability_drill +37 -37
  462. data/scripts/fuzz +63 -63
  463. data/scripts/release +77 -42
  464. data/scripts/release_check +43 -43
  465. data/scripts/replication_failover_drill +268 -250
  466. data/scripts/replication_network_failover_drill +287 -255
  467. data/scripts/restore_drill +45 -45
  468. data/scripts/security +45 -0
  469. metadata +102 -1
@@ -1,387 +1,381 @@
1
- # frozen_string_literal: true
2
-
3
- require "time"
4
- require "set"
5
-
6
- module RubyDB
7
- module Recovery
8
- # CrashRecovery - Handles recovery from crashes
9
- class CrashRecovery
10
- attr_reader :stats, :last_recovery_time
11
-
12
- def initialize(engine, wal, config = {})
13
- @engine = engine
14
- @wal = wal
15
- @config = config
16
- @last_recovery_time = nil
17
- @recovery_log = []
18
- @stats = {
19
- recoveries: 0,
20
- redo_records: 0,
21
- undo_records: 0,
22
- corrupted_records: 0,
23
- skipped_records: 0,
24
- recovery_time_ms: 0,
25
- avg_recovery_time_ms: 0,
26
- total_recovery_time_ms: 0
27
- }
28
- @lock = Mutex.new
29
- @stop_on_error = config.fetch(:stop_on_error, true)
30
- @max_errors = config.fetch(:max_errors, 10)
31
- end
32
-
33
- def recover
34
- @lock.synchronize do
35
- start_time = Time.now
36
- @stats[:recoveries] += 1
37
- @recovery_log.clear
38
- errors = 0
39
-
40
- begin
41
- # Step 1: Find the latest checkpoint
42
- checkpoint_lsn = find_latest_checkpoint
43
- puts "RECOVERY DEBUG: checkpoint_lsn = #{checkpoint_lsn.inspect}" if ENV['DEBUG_RECOVERY']
44
- @recovery_log << { step: "checkpoint", lsn: checkpoint_lsn }
45
-
46
- # Step 2: Read records after checkpoint
47
- records = if checkpoint_lsn
48
- @wal.read_from(checkpoint_lsn)
49
- else
50
- @wal.read_all
51
- end
52
- puts "RECOVERY DEBUG: found #{records.count} WAL records" if ENV['DEBUG_RECOVERY']
53
- records.each { |r| puts " - #{r.type}: #{r.data.inspect}" } if ENV['DEBUG_RECOVERY']
54
- @recovery_log << { step: "read_records", count: records.size }
55
-
56
- # Step 3: Analyze records for redo/undo
57
- analysis = analyze_records(records)
58
- puts "RECOVERY DEBUG: analysis redo=#{analysis[:redo].count}, undo=#{analysis[:undo].count}" if ENV['DEBUG_RECOVERY']
59
- @recovery_log << { step: "analyze", redo: analysis[:redo].size, undo: analysis[:undo].size }
60
-
61
- # Step 4: REDO committed transactions
62
- redo_count = redo_records(analysis[:redo])
63
- @stats[:redo_records] += redo_count
64
- @recovery_log << { step: "redo", count: redo_count }
65
-
66
- # Step 5: UNDO uncommitted transactions
67
- undo_count = undo_records(analysis[:undo])
68
- @stats[:undo_records] += undo_count
69
- @recovery_log << { step: "undo", count: undo_count }
70
-
71
- # Step 6: Verify consistency
72
- consistency_check = verify_consistency
73
- @recovery_log << { step: "consistency", passed: consistency_check }
74
-
75
- # Step 7: Create new checkpoint
76
- if consistency_check
77
- create_checkpoint_after_recovery
78
- end
79
-
80
- @last_recovery_time = Time.now
81
- recovery_time_ms = (Time.now - start_time) * 1000
82
- @stats[:total_recovery_time_ms] += recovery_time_ms
83
- @stats[:recovery_time_ms] = recovery_time_ms
84
- @stats[:avg_recovery_time_ms] = @stats[:total_recovery_time_ms] / @stats[:recoveries]
85
-
86
- {
87
- success: true,
88
- redo_count: redo_count,
89
- undo_count: undo_count,
90
- consistency_passed: consistency_check,
91
- recovery_time_ms: recovery_time_ms,
92
- records_processed: records.size,
93
- log: @recovery_log
94
- }
95
-
96
- rescue => e
97
- @stats[:corrupted_records] += 1
98
- @recovery_log << { step: "error", error: e.message }
99
-
100
- if @stop_on_error || errors >= @max_errors
101
- raise
102
- end
103
-
104
- {
105
- success: false,
106
- error: e.message,
107
- recovery_time_ms: (Time.now - start_time) * 1000,
108
- log: @recovery_log
109
- }
110
- end
111
- end
112
- end
113
-
114
- def find_latest_checkpoint
115
- # Read all records and find the latest checkpoint
116
- all_records = @wal.read_all
117
- checkpoints = all_records.select { |r| r.type == :checkpoint }
118
- return nil if checkpoints.empty?
119
-
120
- latest = checkpoints.max_by { |r| r.timestamp }
121
-
122
- # The checkpoint payload's `lsn` identifies the last WAL record that
123
- # was flushed before the marker was written. It is therefore an
124
- # inclusive boundary: starting recovery there replays that record a
125
- # second time. Use the marker's own physical LSN instead. Reader
126
- # ranges are inclusive, so the marker itself is harmless and all
127
- # records after it are replayed exactly once.
128
- latest.lsn
129
- end
130
-
131
- def analyze_records(records)
132
- analysis = {
133
- redo: [],
134
- undo: [],
135
- committed: Set.new,
136
- prepared: Set.new,
137
- active: Set.new
138
- }
139
-
140
- # First pass: collect transaction states
141
- records.each do |record|
142
- tx_id = record.transaction_id
143
- case record.type
144
- when :commit
145
- analysis[:committed].add(tx_id)
146
- when :prepare
147
- analysis[:prepared].add(tx_id)
148
- when :begin
149
- analysis[:active].add(tx_id)
150
- when :rollback
151
- analysis[:active].delete(tx_id)
152
- end
153
- end
154
-
155
- # Determine which transactions to redo and undo
156
- records.each do |record|
157
- tx_id = record.transaction_id
158
-
159
- # Mutations without explicit transaction markers are auto-committed
160
- if tx_id.nil? || tx_id == 0
161
- if [:insert, :update, :delete, :create_table, :drop_table].include?(record.type)
162
- analysis[:redo] << record
163
- end
164
- elsif analysis[:committed].include?(tx_id)
165
- # Committed transactions need REDO
166
- if [:insert, :update, :delete, :create_table, :drop_table].include?(record.type)
167
- analysis[:redo] << record
168
- end
169
- elsif analysis[:prepared].include?(tx_id)
170
- # Prepared but not committed - check if can commit
171
- if can_commit_prepared?(record)
172
- analysis[:redo] << record
173
- else
174
- analysis[:undo] << record
175
- end
176
- elsif analysis[:active].include?(tx_id)
177
- # Active transactions need UNDO
178
- if [:insert, :update, :delete].include?(record.type)
179
- analysis[:undo] << record
180
- end
181
- end
182
- end
183
-
184
- analysis
185
- end
186
-
187
- def redo_records(records)
188
- @engine.with_recovery do
189
- count = 0
190
- records.each do |record|
191
- begin
192
- redo_record(record)
193
- count += 1
194
- rescue => e
195
- @stats[:corrupted_records] += 1
196
- @recovery_log << { step: "redo_error", record: record.lsn.to_s, error: e.message }
197
- raise if @stop_on_error
198
- end
199
- end
200
- count
201
- end
202
- end
203
-
204
- def undo_records(records)
205
- @engine.with_recovery do
206
- count = 0
207
- # Undo in reverse order
208
- records.reverse_each do |record|
209
- begin
210
- undo_record(record)
211
- count += 1
212
- rescue => e
213
- @stats[:corrupted_records] += 1
214
- @recovery_log << { step: "undo_error", record: record.lsn.to_s, error: e.message }
215
- raise if @stop_on_error
216
- end
217
- end
218
- count
219
- end
220
- end
221
-
222
- def redo_record(record)
223
- data = record.data
224
- case record.type
225
- when :insert
226
- # Re-insert the row
227
- table_name = data[:table_name] || data[:table]
228
- table_name = table_name.to_sym if table_name.respond_to?(:to_sym)
229
- columns = @engine.table_columns(table_name) || []
230
- # JSON-backed WAL payloads may deserialize row_id as a string while
231
- # physical row scans expose an integer. Compare the stable numeric
232
- # identity so replay remains idempotent across process restarts.
233
- existing = @engine.select_rows(table_name, columns).any? do |row|
234
- row[:_row_id].to_i == data[:row_id].to_i
235
- end
236
- return if existing
237
- begin
238
- @engine.insert_row(table_name, columns, data[:values] || {})
239
- rescue => e
240
- warn "REDO INSERT failed: table=#{table_name}, error=#{e.message}"
241
- raise
242
- end
243
- when :update
244
- # Re-apply the update
245
- table_name = data[:table_name] || data[:table]
246
- table_name = table_name.to_sym if table_name.respond_to?(:to_sym)
247
- begin
248
- @engine.update_row(table_name, data[:row_id], data[:values] || {})
249
- rescue => e
250
- warn "REDO UPDATE failed: table=#{table_name}, row=#{data[:row_id]}, error=#{e.message}"
251
- raise
252
- end
253
- when :delete
254
- # Re-delete the row
255
- table_name = data[:table_name] || data[:table]
256
- table_name = table_name.to_sym if table_name.respond_to?(:to_sym)
257
- begin
258
- @engine.delete_row(table_name, data[:row_id])
259
- rescue => e
260
- warn "REDO DELETE failed: table=#{table_name}, row=#{data[:row_id]}, error=#{e.message}"
261
- raise
262
- end
263
- when :create_table
264
- # Re-create the table
265
- begin
266
- @engine.create_table(data[:table_name], data[:columns] || [])
267
- rescue => e
268
- warn "REDO CREATE_TABLE failed: table=#{data[:table_name]}, error=#{e.message}"
269
- raise
270
- end
271
- when :drop_table
272
- # Re-drop the table
273
- begin
274
- @engine.drop_table(data[:table_name])
275
- rescue => e
276
- warn "REDO DROP_TABLE failed: table=#{data[:table_name]}, error=#{e.message}"
277
- raise
278
- end
279
- when :schema_change
280
- # Re-apply schema change
281
- apply_schema_change(data)
282
- end
283
- end
284
-
285
- def undo_record(record)
286
- data = record.data
287
- case record.type
288
- when :insert
289
- # Undo insert: delete the row
290
- table_name = data[:table_name] || data[:table]
291
- table_name = table_name.to_sym if table_name.respond_to?(:to_sym)
292
- @engine.delete_row(table_name, data[:row_id]) if table_name && data[:row_id]
293
- when :update
294
- # Undo update: restore old values
295
- if data[:old_values]
296
- table_name = data[:table_name] || data[:table]
297
- table_name = table_name.to_sym if table_name.respond_to?(:to_sym)
298
- @engine.update_row(table_name, data[:row_id], data[:old_values], visibility_check: false)
299
- end
300
- when :delete
301
- # Undo delete by restoring the original physical row version. This
302
- # preserves its row ID and avoids creating a duplicate row.
303
- table_name = data[:table_name] || data[:table]
304
- table_name = table_name.to_sym if table_name.respond_to?(:to_sym)
305
- @engine.restore_deleted_row(table_name, data[:row_id]) if table_name && data[:row_id]
306
- end
307
- end
308
-
309
- def can_commit_prepared?(record)
310
- # Prepared transactions may commit only when their recorded resources
311
- # remain available after recovery.
312
- data = record.data
313
- return true unless data && data[:resources]
314
-
315
- # Check if all resources are still available
316
- data[:resources].all? do |resource|
317
- resource_available?(resource)
318
- end
319
- end
320
-
321
- def resource_available?(resource)
322
- case resource
323
- when String
324
- File.exist?(resource)
325
- when Hash
326
- path = resource[:path] || resource["path"] || resource[:file] || resource["file"]
327
- path ? File.exist?(path) : false
328
- else
329
- resource.respond_to?(:available?) ? resource.available? : false
330
- end
331
- end
332
-
333
- def apply_schema_change(data)
334
- # Apply schema change
335
- case data[:operation]
336
- when :add_column
337
- table = @engine.find_table(data[:table_name])
338
- table.add_column(data[:column_name], data[:column_type])
339
- when :drop_column
340
- table = @engine.find_table(data[:table_name])
341
- table.drop_column(data[:column_name])
342
- when :rename_table
343
- @engine.rename_table(data[:old_name], data[:new_name])
344
- end
345
- end
346
-
347
- def verify_consistency
348
- # Recovery must not report success when the catalog points at pages
349
- # that cannot be read. This is intentionally structural: row-level
350
- # validation belongs to the storage engine, while recovery needs a
351
- # cheap post-redo/post-undo sanity check.
352
- metadata = @engine.respond_to?(:table_metadata) ? @engine.table_metadata : {}
353
- table_pages = @engine.instance_variable_get(:@table_pages) || {}
354
-
355
- metadata.all? do |_table_name, table_data|
356
- required_pages = [table_data[:metadata_page], table_data[:data_page]]
357
- listed_pages = table_pages.values.flatten
358
- (required_pages + listed_pages).compact.all? do |page_number|
359
- page = @engine.read_page(page_number)
360
- page && page.respond_to?(:header)
361
- rescue StandardError
362
- false
363
- end
364
- end
365
- end
366
-
367
- def create_checkpoint_after_recovery
368
- # Create a checkpoint after successful recovery
369
- @wal.create_checkpoint(true)
370
- end
371
-
372
- def stats
373
- @lock.synchronize do
374
- @stats.merge({
375
- last_recovery: @last_recovery_time&.iso8601,
376
- log_entries: @recovery_log.size,
377
- stop_on_error: @stop_on_error
378
- })
379
- end
380
- end
381
-
382
- def recovery_log
383
- @recovery_log.dup
384
- end
385
- end
386
- end
387
- end
1
+ # frozen_string_literal: true
2
+
3
+ require "time"
4
+
5
+ module RubyDB
6
+ module Recovery
7
+ # CrashRecovery - Handles recovery from crashes
8
+ class CrashRecovery
9
+ attr_reader :stats, :last_recovery_time
10
+
11
+ def initialize(engine, wal, config = {})
12
+ @engine = engine
13
+ @wal = wal
14
+ @config = config
15
+ @last_recovery_time = nil
16
+ @recovery_log = []
17
+ @stats = {
18
+ recoveries: 0,
19
+ redo_records: 0,
20
+ undo_records: 0,
21
+ corrupted_records: 0,
22
+ skipped_records: 0,
23
+ recovery_time_ms: 0,
24
+ avg_recovery_time_ms: 0,
25
+ total_recovery_time_ms: 0
26
+ }
27
+ @lock = Mutex.new
28
+ @stop_on_error = config.fetch(:stop_on_error, true)
29
+ @max_errors = config.fetch(:max_errors, 10)
30
+ end
31
+
32
+ def recover
33
+ @lock.synchronize do
34
+ start_time = Time.now
35
+ @stats[:recoveries] += 1
36
+ @recovery_log.clear
37
+ errors = 0
38
+
39
+ begin
40
+ # Step 1: Find the latest checkpoint
41
+ checkpoint_lsn = find_latest_checkpoint
42
+ puts "RECOVERY DEBUG: checkpoint_lsn = #{checkpoint_lsn.inspect}" if ENV["DEBUG_RECOVERY"]
43
+ @recovery_log << {step: "checkpoint", lsn: checkpoint_lsn}
44
+
45
+ # Step 2: Read records after checkpoint
46
+ records = if checkpoint_lsn
47
+ @wal.read_from(checkpoint_lsn)
48
+ else
49
+ @wal.read_all
50
+ end
51
+ puts "RECOVERY DEBUG: found #{records.count} WAL records" if ENV["DEBUG_RECOVERY"]
52
+ records.each { |r| puts " - #{r.type}: #{r.data.inspect}" } if ENV["DEBUG_RECOVERY"]
53
+ @recovery_log << {step: "read_records", count: records.size}
54
+
55
+ # Step 3: Analyze records for redo/undo
56
+ analysis = analyze_records(records)
57
+ puts "RECOVERY DEBUG: analysis redo=#{analysis[:redo].count}, undo=#{analysis[:undo].count}" if ENV["DEBUG_RECOVERY"]
58
+ @recovery_log << {step: "analyze", redo: analysis[:redo].size, undo: analysis[:undo].size}
59
+
60
+ # Step 4: REDO committed transactions
61
+ redo_count = redo_records(analysis[:redo])
62
+ @stats[:redo_records] += redo_count
63
+ @recovery_log << {step: "redo", count: redo_count}
64
+
65
+ # Step 5: UNDO uncommitted transactions
66
+ undo_count = undo_records(analysis[:undo])
67
+ @stats[:undo_records] += undo_count
68
+ @recovery_log << {step: "undo", count: undo_count}
69
+
70
+ # Step 6: Verify consistency
71
+ consistency_check = verify_consistency
72
+ @recovery_log << {step: "consistency", passed: consistency_check}
73
+
74
+ # Step 7: Create new checkpoint
75
+ if consistency_check
76
+ create_checkpoint_after_recovery
77
+ end
78
+
79
+ @last_recovery_time = Time.now
80
+ recovery_time_ms = (Time.now - start_time) * 1000
81
+ @stats[:total_recovery_time_ms] += recovery_time_ms
82
+ @stats[:recovery_time_ms] = recovery_time_ms
83
+ @stats[:avg_recovery_time_ms] = @stats[:total_recovery_time_ms] / @stats[:recoveries]
84
+
85
+ {
86
+ success: true,
87
+ redo_count: redo_count,
88
+ undo_count: undo_count,
89
+ consistency_passed: consistency_check,
90
+ recovery_time_ms: recovery_time_ms,
91
+ records_processed: records.size,
92
+ log: @recovery_log
93
+ }
94
+ rescue => e
95
+ @stats[:corrupted_records] += 1
96
+ @recovery_log << {step: "error", error: e.message}
97
+
98
+ if @stop_on_error || errors >= @max_errors
99
+ raise
100
+ end
101
+
102
+ {
103
+ success: false,
104
+ error: e.message,
105
+ recovery_time_ms: (Time.now - start_time) * 1000,
106
+ log: @recovery_log
107
+ }
108
+ end
109
+ end
110
+ end
111
+
112
+ def find_latest_checkpoint
113
+ # Read all records and find the latest checkpoint
114
+ all_records = @wal.read_all
115
+ checkpoints = all_records.select { |r| r.type == :checkpoint }
116
+ return nil if checkpoints.empty?
117
+
118
+ latest = checkpoints.max_by { |r| r.timestamp }
119
+
120
+ # The checkpoint payload's `lsn` identifies the last WAL record that
121
+ # was flushed before the marker was written. It is therefore an
122
+ # inclusive boundary: starting recovery there replays that record a
123
+ # second time. Use the marker's own physical LSN instead. Reader
124
+ # ranges are inclusive, so the marker itself is harmless and all
125
+ # records after it are replayed exactly once.
126
+ latest.lsn
127
+ end
128
+
129
+ def analyze_records(records)
130
+ analysis = {
131
+ redo: [],
132
+ undo: [],
133
+ committed: Set.new,
134
+ prepared: Set.new,
135
+ active: Set.new
136
+ }
137
+
138
+ # First pass: collect transaction states
139
+ records.each do |record|
140
+ tx_id = record.transaction_id
141
+ case record.type
142
+ when :commit
143
+ analysis[:committed].add(tx_id)
144
+ when :prepare
145
+ analysis[:prepared].add(tx_id)
146
+ when :begin
147
+ analysis[:active].add(tx_id)
148
+ when :rollback
149
+ analysis[:active].delete(tx_id)
150
+ end
151
+ end
152
+
153
+ # Determine which transactions to redo and undo
154
+ records.each do |record|
155
+ tx_id = record.transaction_id
156
+
157
+ # Mutations without explicit transaction markers are auto-committed
158
+ if tx_id.nil? || tx_id == 0
159
+ if [:insert, :update, :delete, :create_table, :drop_table].include?(record.type)
160
+ analysis[:redo] << record
161
+ end
162
+ elsif analysis[:committed].include?(tx_id)
163
+ # Committed transactions need REDO
164
+ if [:insert, :update, :delete, :create_table, :drop_table].include?(record.type)
165
+ analysis[:redo] << record
166
+ end
167
+ elsif analysis[:prepared].include?(tx_id)
168
+ # Prepared but not committed - check if can commit
169
+ if can_commit_prepared?(record)
170
+ analysis[:redo] << record
171
+ else
172
+ analysis[:undo] << record
173
+ end
174
+ elsif analysis[:active].include?(tx_id)
175
+ # Active transactions need UNDO
176
+ if [:insert, :update, :delete].include?(record.type)
177
+ analysis[:undo] << record
178
+ end
179
+ end
180
+ end
181
+
182
+ analysis
183
+ end
184
+
185
+ def redo_records(records)
186
+ @engine.with_recovery do
187
+ count = 0
188
+ records.each do |record|
189
+ redo_record(record)
190
+ count += 1
191
+ rescue => e
192
+ @stats[:corrupted_records] += 1
193
+ @recovery_log << {step: "redo_error", record: record.lsn.to_s, error: e.message}
194
+ raise if @stop_on_error
195
+ end
196
+ count
197
+ end
198
+ end
199
+
200
+ def undo_records(records)
201
+ @engine.with_recovery do
202
+ count = 0
203
+ # Undo in reverse order
204
+ records.reverse_each do |record|
205
+ undo_record(record)
206
+ count += 1
207
+ rescue => e
208
+ @stats[:corrupted_records] += 1
209
+ @recovery_log << {step: "undo_error", record: record.lsn.to_s, error: e.message}
210
+ raise if @stop_on_error
211
+ end
212
+ count
213
+ end
214
+ end
215
+
216
+ def redo_record(record)
217
+ data = record.data
218
+ case record.type
219
+ when :insert
220
+ # Re-insert the row
221
+ table_name = data[:table_name] || data[:table]
222
+ table_name = table_name.to_sym if table_name.respond_to?(:to_sym)
223
+ columns = @engine.table_columns(table_name) || []
224
+ # JSON-backed WAL payloads may deserialize row_id as a string while
225
+ # physical row scans expose an integer. Compare the stable numeric
226
+ # identity so replay remains idempotent across process restarts.
227
+ existing = @engine.select_rows(table_name, columns).any? do |row|
228
+ row[:_row_id].to_i == data[:row_id].to_i
229
+ end
230
+ return if existing
231
+ begin
232
+ @engine.insert_row(table_name, columns, data[:values] || {})
233
+ rescue => e
234
+ warn "REDO INSERT failed: table=#{table_name}, error=#{e.message}"
235
+ raise
236
+ end
237
+ when :update
238
+ # Re-apply the update
239
+ table_name = data[:table_name] || data[:table]
240
+ table_name = table_name.to_sym if table_name.respond_to?(:to_sym)
241
+ begin
242
+ @engine.update_row(table_name, data[:row_id], data[:values] || {})
243
+ rescue => e
244
+ warn "REDO UPDATE failed: table=#{table_name}, row=#{data[:row_id]}, error=#{e.message}"
245
+ raise
246
+ end
247
+ when :delete
248
+ # Re-delete the row
249
+ table_name = data[:table_name] || data[:table]
250
+ table_name = table_name.to_sym if table_name.respond_to?(:to_sym)
251
+ begin
252
+ @engine.delete_row(table_name, data[:row_id])
253
+ rescue => e
254
+ warn "REDO DELETE failed: table=#{table_name}, row=#{data[:row_id]}, error=#{e.message}"
255
+ raise
256
+ end
257
+ when :create_table
258
+ # Re-create the table
259
+ begin
260
+ @engine.create_table(data[:table_name], data[:columns] || [])
261
+ rescue => e
262
+ warn "REDO CREATE_TABLE failed: table=#{data[:table_name]}, error=#{e.message}"
263
+ raise
264
+ end
265
+ when :drop_table
266
+ # Re-drop the table
267
+ begin
268
+ @engine.drop_table(data[:table_name])
269
+ rescue => e
270
+ warn "REDO DROP_TABLE failed: table=#{data[:table_name]}, error=#{e.message}"
271
+ raise
272
+ end
273
+ when :schema_change
274
+ # Re-apply schema change
275
+ apply_schema_change(data)
276
+ end
277
+ end
278
+
279
+ def undo_record(record)
280
+ data = record.data
281
+ case record.type
282
+ when :insert
283
+ # Undo insert: delete the row
284
+ table_name = data[:table_name] || data[:table]
285
+ table_name = table_name.to_sym if table_name.respond_to?(:to_sym)
286
+ @engine.delete_row(table_name, data[:row_id]) if table_name && data[:row_id]
287
+ when :update
288
+ # Undo update: restore old values
289
+ if data[:old_values]
290
+ table_name = data[:table_name] || data[:table]
291
+ table_name = table_name.to_sym if table_name.respond_to?(:to_sym)
292
+ @engine.update_row(table_name, data[:row_id], data[:old_values], visibility_check: false)
293
+ end
294
+ when :delete
295
+ # Undo delete by restoring the original physical row version. This
296
+ # preserves its row ID and avoids creating a duplicate row.
297
+ table_name = data[:table_name] || data[:table]
298
+ table_name = table_name.to_sym if table_name.respond_to?(:to_sym)
299
+ @engine.restore_deleted_row(table_name, data[:row_id]) if table_name && data[:row_id]
300
+ end
301
+ end
302
+
303
+ def can_commit_prepared?(record)
304
+ # Prepared transactions may commit only when their recorded resources
305
+ # remain available after recovery.
306
+ data = record.data
307
+ return true unless data && data[:resources]
308
+
309
+ # Check if all resources are still available
310
+ data[:resources].all? do |resource|
311
+ resource_available?(resource)
312
+ end
313
+ end
314
+
315
+ def resource_available?(resource)
316
+ case resource
317
+ when String
318
+ File.exist?(resource)
319
+ when Hash
320
+ path = resource[:path] || resource["path"] || resource[:file] || resource["file"]
321
+ path ? File.exist?(path) : false
322
+ else
323
+ resource.respond_to?(:available?) ? resource.available? : false
324
+ end
325
+ end
326
+
327
+ def apply_schema_change(data)
328
+ # Apply schema change
329
+ case data[:operation]
330
+ when :add_column
331
+ table = @engine.find_table(data[:table_name])
332
+ table.add_column(data[:column_name], data[:column_type])
333
+ when :drop_column
334
+ table = @engine.find_table(data[:table_name])
335
+ table.drop_column(data[:column_name])
336
+ when :rename_table
337
+ @engine.rename_table(data[:old_name], data[:new_name])
338
+ end
339
+ end
340
+
341
+ def verify_consistency
342
+ # Recovery must not report success when the catalog points at pages
343
+ # that cannot be read. This is intentionally structural: row-level
344
+ # validation belongs to the storage engine, while recovery needs a
345
+ # cheap post-redo/post-undo sanity check.
346
+ metadata = @engine.respond_to?(:table_metadata) ? @engine.table_metadata : {}
347
+ table_pages = @engine.instance_variable_get(:@table_pages) || {}
348
+
349
+ metadata.all? do |_table_name, table_data|
350
+ required_pages = [table_data[:metadata_page], table_data[:data_page]]
351
+ listed_pages = table_pages.values.flatten
352
+ (required_pages + listed_pages).compact.all? do |page_number|
353
+ page = @engine.read_page(page_number)
354
+ page&.respond_to?(:header)
355
+ rescue
356
+ false
357
+ end
358
+ end
359
+ end
360
+
361
+ def create_checkpoint_after_recovery
362
+ # Create a checkpoint after successful recovery
363
+ @wal.create_checkpoint(true)
364
+ end
365
+
366
+ def stats
367
+ @lock.synchronize do
368
+ @stats.merge({
369
+ last_recovery: @last_recovery_time&.iso8601,
370
+ log_entries: @recovery_log.size,
371
+ stop_on_error: @stop_on_error
372
+ })
373
+ end
374
+ end
375
+
376
+ def recovery_log
377
+ @recovery_log.dup
378
+ end
379
+ end
380
+ end
381
+ end