rubydb 0.1.4 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/PULL_REQUEST_TEMPLATE.md +15 -15
- data/.github/workflows/benchmark.yml +26 -26
- data/.github/workflows/compatibility.yml +63 -63
- data/.github/workflows/fuzz.yml +33 -33
- data/.github/workflows/lint.yml +21 -21
- data/.github/workflows/operations.yml +24 -24
- data/.github/workflows/production-validation.yml +111 -111
- data/.github/workflows/release.yml +77 -77
- data/.github/workflows/security.yml +39 -37
- data/.github/workflows/test.yml +26 -26
- data/.github/workflows/workload.yml +58 -58
- data/.gitignore +16 -5
- data/.rubocop.yml +50 -44
- data/.standard.yml +9 -14
- data/ARCHITECTURE.md +21 -21
- data/CHANGELOG.md +57 -27
- data/CODE_OF_CONDUCT.md +13 -13
- data/CONTRIBUTING.md +29 -29
- data/GOVERNANCE.md +16 -16
- data/Gemfile +18 -17
- data/Gemfile.lock +125 -71
- data/README.md +168 -12
- data/ROADMAP.md +27 -27
- data/Rakefile +76 -71
- data/SECURITY.md +54 -54
- data/SUPPORT.md +14 -14
- data/accelerator/bin/SHA256SUMS +6 -0
- data/accelerator/bin/rubydb-accelerator-darwin-amd64 +0 -0
- data/accelerator/bin/rubydb-accelerator-darwin-arm64 +0 -0
- data/accelerator/bin/rubydb-accelerator-linux-amd64 +0 -0
- data/accelerator/bin/rubydb-accelerator-linux-arm64 +0 -0
- data/accelerator/bin/rubydb-accelerator-windows-amd64.exe +0 -0
- data/accelerator/bin/rubydb-accelerator-windows-arm64.exe +0 -0
- data/accelerator/cmd/rubydb-accelerator/main.go +11 -0
- data/accelerator/go.mod +3 -0
- data/accelerator/internal/execution/aggregate.go +94 -0
- data/accelerator/internal/execution/distinct.go +22 -0
- data/accelerator/internal/execution/filter.go +73 -0
- data/accelerator/internal/execution/join.go +79 -0
- data/accelerator/internal/execution/operators.go +167 -0
- data/accelerator/internal/execution/scan.go +20 -0
- data/accelerator/internal/execution/sort.go +62 -0
- data/accelerator/internal/execution/types.go +136 -0
- data/accelerator/internal/execution/value.go +67 -0
- data/accelerator/internal/memory/arena.go +47 -0
- data/accelerator/internal/memory/reuse.go +22 -0
- data/accelerator/internal/metrics/registry.go +67 -0
- data/accelerator/internal/parallel/bounded_queue.go +56 -0
- data/accelerator/internal/parallel/scheduler.go +47 -0
- data/accelerator/internal/parallel/worker_pool.go +53 -0
- data/accelerator/internal/protocol/cancellation.go +48 -0
- data/accelerator/internal/protocol/columnar.go +263 -0
- data/accelerator/internal/protocol/frame.go +187 -0
- data/accelerator/internal/runtime/worker.go +521 -0
- data/accelerator/internal/storage/page_reader.go +81 -0
- data/accelerator/internal/storage/snapshot_scan.go +539 -0
- data/accelerator/internal/wal/checksum.go +13 -0
- data/accelerator/internal/wal/compression.go +41 -0
- data/accelerator/internal/wal/group_commit.go +24 -0
- data/accelerator/internal/wal/record_encoder.go +40 -0
- data/adapters/activerecord/Gemfile +11 -11
- data/adapters/activerecord/README.md +8 -3
- data/adapters/activerecord/lib/active_record/connection_adapters/rubydb_adapter.rb +881 -879
- data/adapters/activerecord/rubydb-activerecord.gemspec +21 -21
- data/adapters/activerecord/spec/rubydb_adapter_integration_spec.rb +143 -143
- data/adapters/ruby/README.md +18 -18
- data/adapters/sequel/README.md +11 -11
- data/config/monitoring/prometheus-alerts.yml +39 -39
- data/config/production.yml +36 -36
- data/docs/README.md +77 -71
- data/docs/architecture/concurrency.md +14 -14
- data/docs/architecture/current-state.md +125 -125
- data/docs/architecture/execution-engine.md +25 -25
- data/docs/architecture/go-accelerator.md +179 -0
- data/docs/architecture/indexes.md +19 -19
- data/docs/architecture/mvcc.md +19 -19
- data/docs/architecture/overview.md +13 -13
- data/docs/architecture/pages.md +11 -11
- data/docs/architecture/production-roadmap.md +82 -82
- data/docs/architecture/query-planner.md +20 -20
- data/docs/architecture/recovery.md +18 -18
- data/docs/architecture/sql-engine.md +12 -12
- data/docs/architecture/storage-engine.md +14 -14
- data/docs/architecture/transactions.md +10 -10
- data/docs/architecture/wal.md +28 -28
- data/docs/cli-cheatsheet.md +98 -98
- data/docs/cli.md +299 -275
- data/docs/contributing/architecture.md +9 -9
- data/docs/contributing/benchmarking.md +30 -14
- data/docs/contributing/development.md +16 -16
- data/docs/contributing/release-process.md +49 -49
- data/docs/contributing/testing.md +16 -16
- data/docs/debugging.md +229 -229
- data/docs/developer/branching.md +10 -10
- data/docs/developer/database-diff.md +10 -10
- data/docs/developer/local-development.md +49 -17
- data/docs/developer/snapshots.md +9 -9
- data/docs/developer/temporal-data.md +10 -10
- data/docs/developer-guide.md +297 -297
- data/docs/getting-started/first-database.md +16 -16
- data/docs/getting-started/first-query.md +13 -13
- data/docs/getting-started/installation.md +19 -19
- data/docs/getting-started/local-to-production.md +300 -300
- data/docs/getting-started/quickstart.md +17 -17
- data/docs/getting-started/rails.md +16 -16
- data/docs/hardening_backlog.md +93 -93
- data/docs/lessons-learned.md +112 -112
- data/docs/operations/backups.md +33 -33
- data/docs/operations/disaster-recovery.md +31 -31
- data/docs/operations/failover.md +30 -30
- data/docs/operations/monitoring.md +25 -25
- data/docs/operations/production-guide.md +295 -295
- data/docs/operations/production-runbook.md +45 -45
- data/docs/operations/replication.md +33 -33
- data/docs/operations/restore.md +6 -6
- data/docs/operations/runbook.md +34 -34
- data/docs/operations/upgrades.md +14 -14
- data/docs/operations/workload-testing.md +17 -17
- data/docs/production-readiness.md +118 -118
- data/docs/production_validation.md +150 -150
- data/docs/rails/active-record.md +11 -11
- data/docs/rails/compatibility-guide.md +90 -90
- data/docs/rails/database-yml.md +92 -92
- data/docs/rails/installation.md +17 -17
- data/docs/rails/migrations.md +17 -17
- data/docs/rails/production.md +82 -82
- data/docs/rails/troubleshooting.md +18 -18
- data/docs/release.md +25 -25
- data/docs/server/architecture.md +10 -10
- data/docs/server/authentication.md +10 -10
- data/docs/server/configuration.md +16 -16
- data/docs/server/connection-pooling.md +10 -10
- data/docs/server/deployment.md +10 -10
- data/docs/server/protocol.md +12 -12
- data/docs/sql/compatibility-guide.md +82 -82
- data/docs/sql/compatibility.md +39 -39
- data/docs/sql/data-types.md +10 -10
- data/docs/sql/functions.md +9 -9
- data/docs/sql/joins.md +9 -9
- data/docs/sql/operators.md +9 -9
- data/docs/sql/sqlite-compatibility.md +21 -21
- data/docs/sql/syntax.md +10 -10
- data/docs/sql/transactions.md +10 -10
- data/docs/troubleshooting.md +244 -244
- data/lessons/01-foundations.md +73 -0
- data/lessons/02-local-development.md +121 -0
- data/lessons/03-embedded-rubydb.md +99 -0
- data/lessons/04-rails-complex-apps.md +138 -0
- data/lessons/05-rubydb-production-server.md +237 -0
- data/lessons/06-postgresql-massive-apps.md +96 -0
- data/lessons/07-hybrid-microservices.md +179 -0
- data/lessons/08-migrations-backups-recovery.md +86 -0
- data/lessons/09-observability-security-scale.md +87 -0
- data/lessons/10-release-readiness.md +192 -0
- data/lessons/11-community-adapter.md +323 -0
- data/lessons/12-rails-ecommerce-pressure.md +263 -0
- data/lib/rubydb/accelerator/client.rb +451 -0
- data/lib/rubydb/accelerator/error.rb +22 -0
- data/lib/rubydb/accelerator/manager.rb +606 -0
- data/lib/rubydb/accelerator.rb +13 -0
- data/lib/rubydb/backup/archive.rb +332 -334
- data/lib/rubydb/backup/backup.rb +400 -401
- data/lib/rubydb/backup/incremental.rb +349 -353
- data/lib/rubydb/backup/restore.rb +289 -290
- data/lib/rubydb/backup/snapshot.rb +265 -267
- data/lib/rubydb/backup/verification.rb +276 -279
- data/lib/rubydb/branching/branch.rb +181 -181
- data/lib/rubydb/branching/branch_manager.rb +307 -311
- data/lib/rubydb/branching/branch_metadata.rb +140 -140
- data/lib/rubydb/branching/checkout.rb +165 -166
- data/lib/rubydb/branching/copy_on_write.rb +272 -272
- data/lib/rubydb/branching/diff.rb +137 -138
- data/lib/rubydb/branching/merge.rb +282 -285
- data/lib/rubydb/build_info.rb +15 -15
- data/lib/rubydb/catalog/catalog.rb +391 -391
- data/lib/rubydb/catalog/column.rb +112 -112
- data/lib/rubydb/catalog/constraint.rb +180 -180
- data/lib/rubydb/catalog/database.rb +184 -184
- data/lib/rubydb/catalog/index.rb +97 -97
- data/lib/rubydb/catalog/schema.rb +103 -103
- data/lib/rubydb/catalog/sequence.rb +90 -90
- data/lib/rubydb/catalog/system_catalog.rb +698 -698
- data/lib/rubydb/catalog/table.rb +178 -178
- data/lib/rubydb/catalog/trigger.rb +102 -102
- data/lib/rubydb/catalog/view.rb +66 -66
- data/lib/rubydb/cli/application.rb +168 -163
- data/lib/rubydb/cli/commands/accelerator.rb +72 -0
- data/lib/rubydb/cli/commands/backup.rb +80 -81
- data/lib/rubydb/cli/commands/branch.rb +72 -72
- data/lib/rubydb/cli/commands/checkout.rb +54 -54
- data/lib/rubydb/cli/commands/create.rb +58 -58
- data/lib/rubydb/cli/commands/diff.rb +76 -77
- data/lib/rubydb/cli/commands/doctor.rb +77 -74
- data/lib/rubydb/cli/commands/drop.rb +57 -57
- data/lib/rubydb/cli/commands/init.rb +101 -102
- data/lib/rubydb/cli/commands/inspect.rb +95 -95
- data/lib/rubydb/cli/commands/merge.rb +63 -63
- data/lib/rubydb/cli/commands/migrate.rb +62 -62
- data/lib/rubydb/cli/commands/restart.rb +42 -39
- data/lib/rubydb/cli/commands/restore.rb +121 -121
- data/lib/rubydb/cli/commands/shell.rb +365 -365
- data/lib/rubydb/cli/commands/snapshot.rb +79 -79
- data/lib/rubydb/cli/commands/start.rb +88 -82
- data/lib/rubydb/cli/commands/status.rb +96 -92
- data/lib/rubydb/cli/commands/stop.rb +47 -47
- data/lib/rubydb/cli/commands/vacuum.rb +58 -58
- data/lib/rubydb/cli/formatter.rb +221 -221
- data/lib/rubydb/cli/output.rb +168 -168
- data/lib/rubydb/client/client.rb +309 -304
- data/lib/rubydb/client/connection.rb +429 -415
- data/lib/rubydb/client/connection_pool.rb +168 -168
- data/lib/rubydb/client/connection_url.rb +96 -96
- data/lib/rubydb/client/prepared_statement.rb +60 -60
- data/lib/rubydb/client/result.rb +127 -123
- data/lib/rubydb/client/statement.rb +52 -52
- data/lib/rubydb/client/transaction.rb +130 -130
- data/lib/rubydb/concurrency/concurrency.rb +19 -19
- data/lib/rubydb/concurrency/deadlock_detector.rb +148 -150
- data/lib/rubydb/concurrency/latch.rb +101 -101
- data/lib/rubydb/concurrency/lock_graph.rb +163 -165
- data/lib/rubydb/concurrency/mutex.rb +181 -183
- data/lib/rubydb/concurrency/rw_lock.rb +180 -180
- data/lib/rubydb/concurrency/scheduler.rb +248 -250
- data/lib/rubydb/concurrency/worker_pool.rb +145 -143
- data/lib/rubydb/configuration/config.rb +170 -170
- data/lib/rubydb/configuration/defaults.rb +191 -179
- data/lib/rubydb/configuration/environment.rb +152 -152
- data/lib/rubydb/configuration/parser.rb +185 -185
- data/lib/rubydb/configuration/validation.rb +228 -221
- data/lib/rubydb/constants.rb +74 -74
- data/lib/rubydb/constraints/check.rb +181 -181
- data/lib/rubydb/constraints/constraint.rb +101 -101
- data/lib/rubydb/constraints/foreign_key.rb +130 -130
- data/lib/rubydb/constraints/not_null.rb +64 -64
- data/lib/rubydb/constraints/primary_key.rb +99 -99
- data/lib/rubydb/constraints/unique.rb +106 -108
- data/lib/rubydb/constraints/validator.rb +349 -350
- data/lib/rubydb/errors/authentication_error.rb +10 -10
- data/lib/rubydb/errors/authorization_error.rb +23 -23
- data/lib/rubydb/errors/client_error.rb +10 -10
- data/lib/rubydb/errors/configuration_error.rb +10 -10
- data/lib/rubydb/errors/connection_error.rb +10 -10
- data/lib/rubydb/errors/constraint_error.rb +23 -23
- data/lib/rubydb/errors/corruption_error.rb +10 -10
- data/lib/rubydb/errors/database_error.rb +10 -10
- data/lib/rubydb/errors/error.rb +20 -20
- data/lib/rubydb/errors/execution_error.rb +10 -10
- data/lib/rubydb/errors/parser_error.rb +10 -10
- data/lib/rubydb/errors/recovery_error.rb +10 -10
- data/lib/rubydb/errors/replication_error.rb +10 -10
- data/lib/rubydb/errors/server_error.rb +6 -6
- data/lib/rubydb/errors/storage_error.rb +10 -10
- data/lib/rubydb/errors/transaction_error.rb +10 -10
- data/lib/rubydb/execution/accelerator_dispatch.rb +30 -0
- data/lib/rubydb/execution/aggregate_executor.rb +134 -138
- data/lib/rubydb/execution/cost_model.rb +72 -0
- data/lib/rubydb/execution/delete_executor.rb +110 -112
- data/lib/rubydb/execution/distinct_executor.rb +131 -135
- data/lib/rubydb/execution/executor.rb +1544 -1188
- data/lib/rubydb/execution/expression.rb +191 -193
- data/lib/rubydb/execution/index_scan.rb +142 -142
- data/lib/rubydb/execution/insert_executor.rb +215 -217
- data/lib/rubydb/execution/join_executor.rb +243 -249
- data/lib/rubydb/execution/limit_executor.rb +83 -85
- data/lib/rubydb/execution/operator_selection.rb +57 -0
- data/lib/rubydb/execution/optimizer.rb +227 -215
- data/lib/rubydb/execution/physical_plan.rb +47 -0
- data/lib/rubydb/execution/plan.rb +355 -353
- data/lib/rubydb/execution/planner.rb +508 -536
- data/lib/rubydb/execution/predicate.rb +235 -235
- data/lib/rubydb/execution/scan.rb +49 -49
- data/lib/rubydb/execution/sequential_scan.rb +63 -63
- data/lib/rubydb/execution/sort_executor.rb +194 -185
- data/lib/rubydb/execution/update_executor.rb +160 -162
- data/lib/rubydb/functions/aggregate.rb +70 -70
- data/lib/rubydb/functions/date_functions.rb +274 -278
- data/lib/rubydb/functions/function.rb +85 -85
- data/lib/rubydb/functions/json_functions.rb +231 -215
- data/lib/rubydb/functions/numeric_functions.rb +346 -346
- data/lib/rubydb/functions/scalar.rb +52 -52
- data/lib/rubydb/functions/string_functions.rb +383 -383
- data/lib/rubydb/functions/system_functions.rb +258 -246
- data/lib/rubydb/history/as_of.rb +238 -238
- data/lib/rubydb/history/change.rb +105 -105
- data/lib/rubydb/history/history.rb +131 -131
- data/lib/rubydb/history/history_manager.rb +228 -229
- data/lib/rubydb/history/temporal_query.rb +202 -202
- data/lib/rubydb/history/timeline.rb +144 -144
- data/lib/rubydb/indexes/btree.rb +215 -186
- data/lib/rubydb/indexes/btree_cursor.rb +258 -258
- data/lib/rubydb/indexes/btree_node.rb +384 -385
- data/lib/rubydb/indexes/hash_index.rb +150 -150
- data/lib/rubydb/indexes/index.rb +71 -71
- data/lib/rubydb/indexes/index_manager.rb +408 -406
- data/lib/rubydb/indexes/index_scan.rb +466 -470
- data/lib/rubydb/migrations/migration.rb +253 -254
- data/lib/rubydb/migrations/migration_lock.rb +146 -146
- data/lib/rubydb/migrations/migration_manager.rb +187 -176
- data/lib/rubydb/migrations/migration_version.rb +71 -71
- data/lib/rubydb/migrations/schema_diff.rb +211 -211
- data/lib/rubydb/migrations/schema_version.rb +64 -64
- data/lib/rubydb/monitoring/events.rb +155 -160
- data/lib/rubydb/monitoring/health.rb +216 -222
- data/lib/rubydb/monitoring/logger.rb +188 -193
- data/lib/rubydb/monitoring/metrics.rb +363 -359
- data/lib/rubydb/monitoring/performance.rb +176 -176
- data/lib/rubydb/monitoring/statistics.rb +168 -170
- data/lib/rubydb/mvcc/garbage_collector.rb +199 -199
- data/lib/rubydb/mvcc/mvcc.rb +16 -16
- data/lib/rubydb/mvcc/snapshot.rb +146 -147
- data/lib/rubydb/mvcc/vacuum.rb +180 -180
- data/lib/rubydb/mvcc/version.rb +106 -106
- data/lib/rubydb/mvcc/version_store.rb +396 -398
- data/lib/rubydb/mvcc/visibility.rb +107 -109
- data/lib/rubydb/protocol/capabilities.rb +125 -125
- data/lib/rubydb/protocol/decoder.rb +142 -145
- data/lib/rubydb/protocol/encoder.rb +131 -136
- data/lib/rubydb/protocol/handshake.rb +306 -305
- data/lib/rubydb/protocol/message.rb +121 -121
- data/lib/rubydb/protocol/parameter_binder.rb +101 -0
- data/lib/rubydb/protocol/protocol.rb +276 -277
- data/lib/rubydb/protocol/version.rb +54 -54
- data/lib/rubydb/rails/adapter.rb +245 -239
- data/lib/rubydb/rails/connection.rb +312 -314
- data/lib/rubydb/rails/database_statements.rb +122 -122
- data/lib/rubydb/rails/migration.rb +131 -131
- data/lib/rubydb/rails/quoting.rb +109 -109
- data/lib/rubydb/rails/result.rb +117 -117
- data/lib/rubydb/rails/schema_statements.rb +339 -339
- data/lib/rubydb/rails/transaction.rb +105 -105
- data/lib/rubydb/rails/type.rb +126 -126
- data/lib/rubydb/recovery/checkpoint.rb +261 -257
- data/lib/rubydb/recovery/consistency.rb +457 -467
- data/lib/rubydb/recovery/corruption_detector.rb +5 -5
- data/lib/rubydb/recovery/crash_recovery.rb +381 -387
- data/lib/rubydb/recovery/recovery_manager.rb +204 -206
- data/lib/rubydb/recovery/redo.rb +235 -237
- data/lib/rubydb/recovery/undo.rb +204 -206
- data/lib/rubydb/replication/failover.rb +5 -5
- data/lib/rubydb/replication/fencing.rb +63 -63
- data/lib/rubydb/replication/primary.rb +461 -450
- data/lib/rubydb/replication/replica.rb +382 -384
- data/lib/rubydb/replication/replication_log.rb +194 -200
- data/lib/rubydb/replication/replication_manager.rb +307 -308
- data/lib/rubydb/replication/replication_slot.rb +293 -295
- data/lib/rubydb/replication/replication_stream.rb +198 -201
- data/lib/rubydb/rubydb.rb +570 -560
- data/lib/rubydb/security/access_control.rb +252 -254
- data/lib/rubydb/security/audit_log.rb +209 -213
- data/lib/rubydb/security/authentication.rb +302 -302
- data/lib/rubydb/security/authorization.rb +282 -282
- data/lib/rubydb/security/credentials.rb +192 -196
- data/lib/rubydb/security/password.rb +205 -215
- data/lib/rubydb/security/permissions.rb +74 -74
- data/lib/rubydb/security/role.rb +99 -101
- data/lib/rubydb/security/user.rb +86 -86
- data/lib/rubydb/server/connection.rb +383 -366
- data/lib/rubydb/server/connection_pool.rb +193 -193
- data/lib/rubydb/server/lifecycle.rb +227 -228
- data/lib/rubydb/server/listener.rb +139 -136
- data/lib/rubydb/server/request_handler.rb +277 -276
- data/lib/rubydb/server/server.rb +363 -364
- data/lib/rubydb/server/session.rb +416 -369
- data/lib/rubydb/server/worker.rb +206 -210
- data/lib/rubydb/server/worker_pool.rb +168 -168
- data/lib/rubydb/sql/ast/alter_table.rb +169 -169
- data/lib/rubydb/sql/ast/begin_transaction.rb +47 -47
- data/lib/rubydb/sql/ast/commit.rb +37 -37
- data/lib/rubydb/sql/ast/constraint.rb +92 -83
- data/lib/rubydb/sql/ast/create_database.rb +41 -41
- data/lib/rubydb/sql/ast/create_index.rb +61 -61
- data/lib/rubydb/sql/ast/create_schema.rb +52 -52
- data/lib/rubydb/sql/ast/create_table.rb +187 -187
- data/lib/rubydb/sql/ast/delete.rb +54 -54
- data/lib/rubydb/sql/ast/drop_database.rb +41 -41
- data/lib/rubydb/sql/ast/drop_index.rb +41 -41
- data/lib/rubydb/sql/ast/drop_schema.rb +49 -49
- data/lib/rubydb/sql/ast/drop_table.rb +49 -49
- data/lib/rubydb/sql/ast/explain.rb +64 -64
- data/lib/rubydb/sql/ast/expression.rb +617 -604
- data/lib/rubydb/sql/ast/insert.rb +66 -66
- data/lib/rubydb/sql/ast/node.rb +42 -42
- data/lib/rubydb/sql/ast/rollback.rb +63 -63
- data/lib/rubydb/sql/ast/savepoint.rb +59 -59
- data/lib/rubydb/sql/ast/select.rb +88 -88
- data/lib/rubydb/sql/ast/set_operation.rb +22 -20
- data/lib/rubydb/sql/ast/trigger.rb +35 -29
- data/lib/rubydb/sql/ast/update.rb +88 -88
- data/lib/rubydb/sql/ast/vacuum.rb +19 -19
- data/lib/rubydb/sql/ast/view.rb +38 -32
- data/lib/rubydb/sql/ast/with.rb +32 -32
- data/lib/rubydb/sql/grammar.rb +86 -86
- data/lib/rubydb/sql/keywords.rb +156 -156
- data/lib/rubydb/sql/lexer.rb +209 -214
- data/lib/rubydb/sql/operators.rb +100 -100
- data/lib/rubydb/sql/parser.rb +1167 -1170
- data/lib/rubydb/sql/planner/analyzer.rb +283 -302
- data/lib/rubydb/sql/planner/binder.rb +537 -550
- data/lib/rubydb/sql/planner/type_checker.rb +427 -431
- data/lib/rubydb/sql/token.rb +210 -210
- data/lib/rubydb/storage/buffer_frame.rb +44 -44
- data/lib/rubydb/storage/buffer_pool.rb +155 -155
- data/lib/rubydb/storage/database_lock.rb +74 -74
- data/lib/rubydb/storage/deserializer.rb +332 -342
- data/lib/rubydb/storage/engine.rb +2409 -2330
- data/lib/rubydb/storage/file_manager.rb +191 -187
- data/lib/rubydb/storage/free_space_map.rb +79 -81
- data/lib/rubydb/storage/page.rb +92 -94
- data/lib/rubydb/storage/page_allocator.rb +852 -855
- data/lib/rubydb/storage/page_header.rb +63 -67
- data/lib/rubydb/storage/page_manager.rb +127 -131
- data/lib/rubydb/storage/record.rb +58 -58
- data/lib/rubydb/storage/row.rb +78 -78
- data/lib/rubydb/storage/serializer.rb +51 -51
- data/lib/rubydb/storage/snapshot_reader.rb +167 -0
- data/lib/rubydb/storage/storage_layout.rb +151 -151
- data/lib/rubydb/storage/storage_manager.rb +114 -114
- data/lib/rubydb/storage/tuple.rb +458 -461
- data/lib/rubydb/storage/visibility_map.rb +964 -973
- data/lib/rubydb/transactions/commit_manager.rb +219 -220
- data/lib/rubydb/transactions/isolation.rb +98 -98
- data/lib/rubydb/transactions/lock.rb +76 -76
- data/lib/rubydb/transactions/lock_manager.rb +359 -362
- data/lib/rubydb/transactions/savepoint.rb +142 -143
- data/lib/rubydb/transactions/transaction.rb +214 -215
- data/lib/rubydb/transactions/transaction_id.rb +84 -84
- data/lib/rubydb/transactions/transaction_log.rb +256 -257
- data/lib/rubydb/transactions/transaction_manager.rb +434 -435
- data/lib/rubydb/types/bigint.rb +36 -36
- data/lib/rubydb/types/blob.rb +37 -37
- data/lib/rubydb/types/boolean.rb +34 -34
- data/lib/rubydb/types/date.rb +39 -39
- data/lib/rubydb/types/decimal.rb +48 -48
- data/lib/rubydb/types/float.rb +34 -34
- data/lib/rubydb/types/integer.rb +36 -36
- data/lib/rubydb/types/json.rb +41 -41
- data/lib/rubydb/types/null.rb +34 -34
- data/lib/rubydb/types/smallint.rb +36 -36
- data/lib/rubydb/types/text.rb +37 -37
- data/lib/rubydb/types/time.rb +46 -46
- data/lib/rubydb/types/timestamp.rb +39 -39
- data/lib/rubydb/types/type.rb +119 -119
- data/lib/rubydb/types/uuid.rb +47 -47
- data/lib/rubydb/types/varchar.rb +37 -37
- data/lib/rubydb/version.rb +32 -32
- data/lib/rubydb/wal/archive.rb +207 -193
- data/lib/rubydb/wal/checkpoint.rb +181 -183
- data/lib/rubydb/wal/lsn.rb +94 -94
- data/lib/rubydb/wal/reader.rb +259 -260
- data/lib/rubydb/wal/record.rb +105 -105
- data/lib/rubydb/wal/segment.rb +193 -193
- data/lib/rubydb/wal/wal.rb +481 -452
- data/lib/rubydb/wal/writer.rb +236 -236
- data/lib/rubydb.rb +7 -7
- data/packaging/docker/docker-compose.failover.yml +43 -43
- data/packaging/homebrew/rubydb.rb +19 -19
- data/rubydb.gemspec +70 -57
- data/scripts/benchmark +7 -7
- data/scripts/build_accelerator +49 -0
- data/scripts/durability_drill +37 -37
- data/scripts/fuzz +63 -63
- data/scripts/release +77 -42
- data/scripts/release_check +43 -43
- data/scripts/replication_failover_drill +268 -250
- data/scripts/replication_network_failover_drill +287 -255
- data/scripts/restore_drill +45 -45
- data/scripts/security +45 -0
- metadata +102 -1
|
@@ -0,0 +1,539 @@
|
|
|
1
|
+
package storage
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"context"
|
|
5
|
+
"crypto/sha256"
|
|
6
|
+
"encoding/binary"
|
|
7
|
+
"encoding/hex"
|
|
8
|
+
"encoding/json"
|
|
9
|
+
"errors"
|
|
10
|
+
"fmt"
|
|
11
|
+
"io"
|
|
12
|
+
"math"
|
|
13
|
+
"os"
|
|
14
|
+
"strings"
|
|
15
|
+
"time"
|
|
16
|
+
|
|
17
|
+
"github.com/aldanedev-create/rubydb/accelerator/internal/execution"
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
const (
|
|
21
|
+
snapshotFormatVersion = 1
|
|
22
|
+
pageHeaderSize = 64
|
|
23
|
+
recordHeaderSize = 16
|
|
24
|
+
nullBitmapFlag = 0x02
|
|
25
|
+
variablePrefixFlag = 0x04
|
|
26
|
+
deletedRecordFlag = 0x01
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
var (
|
|
30
|
+
ErrInvalidSnapshot = errors.New("invalid immutable storage snapshot")
|
|
31
|
+
ErrSnapshotSchema = errors.New("immutable snapshot schema is invalid")
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
type SnapshotManifest struct {
|
|
35
|
+
FormatVersion int `json:"format_version"`
|
|
36
|
+
SnapshotID string `json:"snapshot_id"`
|
|
37
|
+
SnapshotPath string `json:"snapshot_path"`
|
|
38
|
+
PageSize int `json:"page_size"`
|
|
39
|
+
PageCount uint64 `json:"page_count"`
|
|
40
|
+
FileSHA256 string `json:"file_sha256"`
|
|
41
|
+
Tables map[string]SnapshotTable `json:"tables"`
|
|
42
|
+
HiddenRowIDs []uint64 `json:"hidden_row_ids"`
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
type SnapshotTable struct {
|
|
46
|
+
Pages []uint64 `json:"pages"`
|
|
47
|
+
Columns []SnapshotColumn `json:"columns"`
|
|
48
|
+
Indexes []SnapshotIndex `json:"indexes"`
|
|
49
|
+
RowCount int `json:"row_count"`
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
type SnapshotColumn struct {
|
|
53
|
+
Name string `json:"name"`
|
|
54
|
+
Type string `json:"type"`
|
|
55
|
+
Nullable bool `json:"nullable"`
|
|
56
|
+
Default interface{} `json:"default,omitempty"`
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
type SnapshotIndex struct {
|
|
60
|
+
Name string `json:"name"`
|
|
61
|
+
Type string `json:"type"`
|
|
62
|
+
Columns []string `json:"columns"`
|
|
63
|
+
Unique bool `json:"unique"`
|
|
64
|
+
Entries []SnapshotEntry `json:"entries"`
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
type SnapshotEntry struct {
|
|
68
|
+
Key interface{} `json:"key"`
|
|
69
|
+
RowID uint64 `json:"row_id"`
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
type SnapshotScanRequest struct {
|
|
73
|
+
Snapshot SnapshotManifest `json:"snapshot"`
|
|
74
|
+
Table string `json:"table"`
|
|
75
|
+
Columns []string `json:"columns"`
|
|
76
|
+
ColumnTypes map[string]string `json:"column_types,omitempty"`
|
|
77
|
+
Filters []execution.Filter `json:"filters,omitempty"`
|
|
78
|
+
OrderBy []execution.Order `json:"order_by,omitempty"`
|
|
79
|
+
IndexName string `json:"index_name,omitempty"`
|
|
80
|
+
Limit *int `json:"limit,omitempty"`
|
|
81
|
+
Offset int `json:"offset,omitempty"`
|
|
82
|
+
Distinct bool `json:"distinct,omitempty"`
|
|
83
|
+
DistinctColumns []string `json:"distinct_columns,omitempty"`
|
|
84
|
+
BatchSize int `json:"batch_size,omitempty"`
|
|
85
|
+
MaxRows int `json:"max_rows,omitempty"`
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
type SnapshotScanResult struct {
|
|
89
|
+
Rows []map[string]interface{}
|
|
90
|
+
ColumnTypes map[string]string
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// ExecuteSnapshotScan validates and reads only the file named by the
|
|
94
|
+
// lock-protected manifest. In direct mode that is the already-flushed live
|
|
95
|
+
// file; in detached mode it is a short-lived copy. It never consults WAL or
|
|
96
|
+
// MVCC state; Ruby has already established the eligible read snapshot.
|
|
97
|
+
func ExecuteSnapshotScan(request SnapshotScanRequest) (SnapshotScanResult, error) {
|
|
98
|
+
return ExecuteSnapshotScanContext(context.Background(), request)
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
// ExecuteSnapshotScanContext is cancellation-aware and stops page work as
|
|
102
|
+
// soon as the request context is cancelled. Ruby owns snapshot creation and
|
|
103
|
+
// visibility; Go only reads the file named by the manifest while Ruby holds
|
|
104
|
+
// the engine snapshot lock.
|
|
105
|
+
func ExecuteSnapshotScanContext(ctx context.Context, request SnapshotScanRequest) (SnapshotScanResult, error) {
|
|
106
|
+
if err := validateManifest(request.Snapshot); err != nil {
|
|
107
|
+
return SnapshotScanResult{}, err
|
|
108
|
+
}
|
|
109
|
+
file, err := os.Open(request.Snapshot.SnapshotPath)
|
|
110
|
+
if err != nil {
|
|
111
|
+
return SnapshotScanResult{}, fmt.Errorf("open snapshot: %w", err)
|
|
112
|
+
}
|
|
113
|
+
defer file.Close()
|
|
114
|
+
|
|
115
|
+
if err := validateSnapshotFile(file, request.Snapshot); err != nil {
|
|
116
|
+
return SnapshotScanResult{}, err
|
|
117
|
+
}
|
|
118
|
+
table, ok := findTable(request.Snapshot.Tables, request.Table)
|
|
119
|
+
if !ok || len(table.Columns) == 0 {
|
|
120
|
+
return SnapshotScanResult{}, fmt.Errorf("%w: table %q", ErrSnapshotSchema, request.Table)
|
|
121
|
+
}
|
|
122
|
+
columns, err := selectedColumns(table.Columns, request.Columns)
|
|
123
|
+
if err != nil {
|
|
124
|
+
return SnapshotScanResult{}, err
|
|
125
|
+
}
|
|
126
|
+
hidden := make(map[uint64]struct{}, len(request.Snapshot.HiddenRowIDs))
|
|
127
|
+
for _, rowID := range request.Snapshot.HiddenRowIDs {
|
|
128
|
+
hidden[rowID] = struct{}{}
|
|
129
|
+
}
|
|
130
|
+
indexed, useIndex, err := indexedRowIDs(table.Indexes, request)
|
|
131
|
+
if err != nil {
|
|
132
|
+
return SnapshotScanResult{}, err
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
rows := make([]map[string]interface{}, 0, table.RowCount)
|
|
136
|
+
needed := -1
|
|
137
|
+
if len(request.OrderBy) == 0 && !request.Distinct && request.Limit != nil {
|
|
138
|
+
if *request.Limit < 0 || request.Offset < 0 {
|
|
139
|
+
return SnapshotScanResult{}, execution.ErrInvalidWindow("limit and offset cannot be negative")
|
|
140
|
+
}
|
|
141
|
+
needed = request.Offset + *request.Limit
|
|
142
|
+
}
|
|
143
|
+
for _, pageNumber := range table.Pages {
|
|
144
|
+
select {
|
|
145
|
+
case <-ctx.Done():
|
|
146
|
+
return SnapshotScanResult{}, ctx.Err()
|
|
147
|
+
default:
|
|
148
|
+
}
|
|
149
|
+
page, err := readSnapshotPage(file, request.Snapshot, pageNumber)
|
|
150
|
+
if err != nil {
|
|
151
|
+
return SnapshotScanResult{}, err
|
|
152
|
+
}
|
|
153
|
+
pageRows, err := scanPage(page, table.Columns, columns, hidden, indexed, useIndex, request.Filters)
|
|
154
|
+
if err != nil {
|
|
155
|
+
return SnapshotScanResult{}, fmt.Errorf("scan page %d: %w", pageNumber, err)
|
|
156
|
+
}
|
|
157
|
+
rows = append(rows, pageRows...)
|
|
158
|
+
if request.MaxRows > 0 && len(rows) > request.MaxRows {
|
|
159
|
+
return SnapshotScanResult{}, fmt.Errorf("snapshot scan exceeds configured row limit")
|
|
160
|
+
}
|
|
161
|
+
if needed >= 0 && len(rows) >= needed {
|
|
162
|
+
break
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
if len(request.OrderBy) > 0 {
|
|
167
|
+
execution.SortRows(rows, request.OrderBy)
|
|
168
|
+
}
|
|
169
|
+
if request.Distinct {
|
|
170
|
+
columns := request.DistinctColumns
|
|
171
|
+
if len(columns) == 0 {
|
|
172
|
+
columns = request.Columns
|
|
173
|
+
}
|
|
174
|
+
rows = execution.DistinctRows(rows, columns)
|
|
175
|
+
}
|
|
176
|
+
rows = applyWindow(rows, request.Offset, request.Limit)
|
|
177
|
+
columnTypes := make(map[string]string, len(columns))
|
|
178
|
+
for _, column := range columns {
|
|
179
|
+
columnTypes[column.Name] = column.Type
|
|
180
|
+
}
|
|
181
|
+
return SnapshotScanResult{Rows: rows, ColumnTypes: columnTypes}, nil
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
func validateManifest(manifest SnapshotManifest) error {
|
|
185
|
+
if manifest.FormatVersion != snapshotFormatVersion || manifest.SnapshotID == "" || manifest.SnapshotPath == "" {
|
|
186
|
+
return ErrInvalidSnapshot
|
|
187
|
+
}
|
|
188
|
+
if manifest.PageSize < pageHeaderSize || manifest.PageSize > 64*1024*1024 || manifest.PageCount == 0 {
|
|
189
|
+
return ErrInvalidSnapshot
|
|
190
|
+
}
|
|
191
|
+
if len(manifest.Tables) == 0 {
|
|
192
|
+
return ErrInvalidSnapshot
|
|
193
|
+
}
|
|
194
|
+
if manifest.FileSHA256 != "" {
|
|
195
|
+
if len(manifest.FileSHA256) != sha256.Size*2 {
|
|
196
|
+
return ErrInvalidSnapshot
|
|
197
|
+
}
|
|
198
|
+
if _, err := hex.DecodeString(manifest.FileSHA256); err != nil {
|
|
199
|
+
return ErrInvalidSnapshot
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
return nil
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
func validateSnapshotFile(file *os.File, manifest SnapshotManifest) error {
|
|
206
|
+
info, err := file.Stat()
|
|
207
|
+
if err != nil || info.Size() != int64(manifest.PageSize)*int64(manifest.PageCount) {
|
|
208
|
+
return ErrInvalidSnapshot
|
|
209
|
+
}
|
|
210
|
+
if manifest.FileSHA256 != "" {
|
|
211
|
+
if _, err := file.Seek(0, io.SeekStart); err != nil {
|
|
212
|
+
return ErrInvalidSnapshot
|
|
213
|
+
}
|
|
214
|
+
digest := sha256.New()
|
|
215
|
+
if _, err := io.Copy(digest, file); err != nil || !strings.EqualFold(hex.EncodeToString(digest.Sum(nil)), manifest.FileSHA256) {
|
|
216
|
+
return ErrInvalidSnapshot
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
page, err := readSnapshotPage(file, manifest, 0)
|
|
220
|
+
if err != nil || binary.BigEndian.Uint64(page[0:8]) != 0 || binary.BigEndian.Uint64(page[8:16]) != uint64(manifest.PageSize) ||
|
|
221
|
+
binary.BigEndian.Uint32(page[16:20]) != pageHeaderSize || binary.BigEndian.Uint32(page[32:36]) != 1 {
|
|
222
|
+
return ErrInvalidSnapshot
|
|
223
|
+
}
|
|
224
|
+
return nil
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
func readSnapshotPage(file *os.File, manifest SnapshotManifest, pageNumber uint64) ([]byte, error) {
|
|
228
|
+
if pageNumber >= manifest.PageCount {
|
|
229
|
+
return nil, ErrInvalidSnapshot
|
|
230
|
+
}
|
|
231
|
+
page := make([]byte, manifest.PageSize)
|
|
232
|
+
read, err := file.ReadAt(page, int64(pageNumber)*int64(manifest.PageSize))
|
|
233
|
+
if read != len(page) || (err != nil && !errors.Is(err, io.EOF)) {
|
|
234
|
+
return nil, ErrInvalidSnapshot
|
|
235
|
+
}
|
|
236
|
+
if binary.BigEndian.Uint64(page[0:8]) != pageNumber || binary.BigEndian.Uint64(page[8:16]) != uint64(manifest.PageSize) {
|
|
237
|
+
return nil, ErrInvalidSnapshot
|
|
238
|
+
}
|
|
239
|
+
dataEnd := binary.BigEndian.Uint32(page[20:24])
|
|
240
|
+
if dataEnd < pageHeaderSize || dataEnd > uint32(len(page)) {
|
|
241
|
+
return nil, ErrInvalidSnapshot
|
|
242
|
+
}
|
|
243
|
+
return page, nil
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
func findTable(tables map[string]SnapshotTable, name string) (SnapshotTable, bool) {
|
|
247
|
+
if table, ok := tables[name]; ok {
|
|
248
|
+
return table, true
|
|
249
|
+
}
|
|
250
|
+
for tableName, table := range tables {
|
|
251
|
+
if strings.EqualFold(tableName, name) {
|
|
252
|
+
return table, true
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
return SnapshotTable{}, false
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
func selectedColumns(all []SnapshotColumn, requested []string) ([]SnapshotColumn, error) {
|
|
259
|
+
if len(requested) == 0 {
|
|
260
|
+
return all, nil
|
|
261
|
+
}
|
|
262
|
+
byName := make(map[string]SnapshotColumn, len(all))
|
|
263
|
+
for _, column := range all {
|
|
264
|
+
byName[strings.ToLower(column.Name)] = column
|
|
265
|
+
}
|
|
266
|
+
selected := make([]SnapshotColumn, 0, len(requested))
|
|
267
|
+
for _, name := range requested {
|
|
268
|
+
column, ok := byName[strings.ToLower(name)]
|
|
269
|
+
if !ok {
|
|
270
|
+
return nil, fmt.Errorf("%w: unknown column %q", ErrSnapshotSchema, name)
|
|
271
|
+
}
|
|
272
|
+
selected = append(selected, column)
|
|
273
|
+
}
|
|
274
|
+
return selected, nil
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
func scanPage(page []byte, schema, selected []SnapshotColumn, hidden, indexed map[uint64]struct{}, useIndex bool, filters []execution.Filter) ([]map[string]interface{}, error) {
|
|
278
|
+
dataEnd := int(binary.BigEndian.Uint32(page[20:24]))
|
|
279
|
+
rows := make([]map[string]interface{}, 0)
|
|
280
|
+
for offset := pageHeaderSize; offset < dataEnd; {
|
|
281
|
+
if dataEnd-offset < recordHeaderSize {
|
|
282
|
+
return nil, fmt.Errorf("record header at offset %d: %w", offset, ErrInvalidSnapshot)
|
|
283
|
+
}
|
|
284
|
+
rowID := binary.BigEndian.Uint64(page[offset : offset+8])
|
|
285
|
+
recordSize := int(binary.BigEndian.Uint32(page[offset+8 : offset+12]))
|
|
286
|
+
// Ruby's record header is Q>L>S>S: the flags field is explicitly
|
|
287
|
+
// big-endian while the final column-count S is native-endian.
|
|
288
|
+
flags := binary.BigEndian.Uint16(page[offset+12 : offset+14])
|
|
289
|
+
columnCount := int(binary.LittleEndian.Uint16(page[offset+14 : offset+16]))
|
|
290
|
+
offset += recordHeaderSize
|
|
291
|
+
if recordSize < 0 || offset+recordSize > dataEnd || columnCount < 0 || columnCount > len(schema) {
|
|
292
|
+
return nil, fmt.Errorf("record at offset %d has size %d and %d columns: %w", offset-recordHeaderSize, recordSize, columnCount, ErrInvalidSnapshot)
|
|
293
|
+
}
|
|
294
|
+
record := page[offset : offset+recordSize]
|
|
295
|
+
offset += recordSize
|
|
296
|
+
if flags&deletedRecordFlag != 0 {
|
|
297
|
+
continue
|
|
298
|
+
}
|
|
299
|
+
if _, ok := hidden[rowID]; ok {
|
|
300
|
+
continue
|
|
301
|
+
}
|
|
302
|
+
if useIndex {
|
|
303
|
+
if _, ok := indexed[rowID]; !ok {
|
|
304
|
+
continue
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
row, err := decodeRow(record, flags, columnCount, schema, selected)
|
|
308
|
+
if err != nil {
|
|
309
|
+
return nil, fmt.Errorf("row %d: %w", rowID, err)
|
|
310
|
+
}
|
|
311
|
+
row["_row_id"] = rowID
|
|
312
|
+
if !matchesFilters(row, filters) {
|
|
313
|
+
continue
|
|
314
|
+
}
|
|
315
|
+
rows = append(rows, row)
|
|
316
|
+
}
|
|
317
|
+
return rows, nil
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
func decodeRow(data []byte, flags uint16, columnCount int, schema, selected []SnapshotColumn) (map[string]interface{}, error) {
|
|
321
|
+
bitmapSize := 0
|
|
322
|
+
if flags&nullBitmapFlag != 0 {
|
|
323
|
+
bitmapSize = (len(schema) + 7) / 8
|
|
324
|
+
if bitmapSize > len(data) {
|
|
325
|
+
return nil, ErrInvalidSnapshot
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
bitmap := data[:bitmapSize]
|
|
329
|
+
offset := bitmapSize
|
|
330
|
+
selectedNames := make(map[string]struct{}, len(selected))
|
|
331
|
+
for _, column := range selected {
|
|
332
|
+
selectedNames[strings.ToLower(column.Name)] = struct{}{}
|
|
333
|
+
}
|
|
334
|
+
row := make(map[string]interface{}, len(selected))
|
|
335
|
+
for index := 0; index < len(schema); index++ {
|
|
336
|
+
column := schema[index]
|
|
337
|
+
if index >= columnCount {
|
|
338
|
+
if _, ok := selectedNames[strings.ToLower(column.Name)]; ok {
|
|
339
|
+
row[column.Name] = column.Default
|
|
340
|
+
}
|
|
341
|
+
continue
|
|
342
|
+
}
|
|
343
|
+
value, next, err := decodeColumn(data, offset, column.Type, flags&variablePrefixFlag != 0)
|
|
344
|
+
if err != nil {
|
|
345
|
+
return nil, fmt.Errorf("column %q: %w", column.Name, err)
|
|
346
|
+
}
|
|
347
|
+
offset = next
|
|
348
|
+
if bitmapSet(bitmap, index) {
|
|
349
|
+
value = nil
|
|
350
|
+
}
|
|
351
|
+
if value == nil && column.Default != nil {
|
|
352
|
+
value = column.Default
|
|
353
|
+
}
|
|
354
|
+
if _, ok := selectedNames[strings.ToLower(column.Name)]; ok {
|
|
355
|
+
row[column.Name] = value
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
if offset > len(data) {
|
|
359
|
+
return nil, ErrInvalidSnapshot
|
|
360
|
+
}
|
|
361
|
+
return row, nil
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
func decodeColumn(data []byte, offset int, typeName string, variablePrefixes bool) (interface{}, int, error) {
|
|
365
|
+
typeName = strings.ToLower(typeName)
|
|
366
|
+
size := fixedSize(typeName)
|
|
367
|
+
if size > 0 {
|
|
368
|
+
if offset+size > len(data) {
|
|
369
|
+
return nil, offset, ErrInvalidSnapshot
|
|
370
|
+
}
|
|
371
|
+
raw := data[offset : offset+size]
|
|
372
|
+
return decodeFixed(raw, typeName), offset + size, nil
|
|
373
|
+
}
|
|
374
|
+
if !variablePrefixes {
|
|
375
|
+
return nil, len(data), nil
|
|
376
|
+
}
|
|
377
|
+
if offset+4 > len(data) {
|
|
378
|
+
return nil, offset, ErrInvalidSnapshot
|
|
379
|
+
}
|
|
380
|
+
length := int(binary.BigEndian.Uint32(data[offset : offset+4]))
|
|
381
|
+
offset += 4
|
|
382
|
+
if length < 0 || offset+length > len(data) {
|
|
383
|
+
return nil, offset, ErrInvalidSnapshot
|
|
384
|
+
}
|
|
385
|
+
return decodeVariable(data[offset:offset+length], typeName), offset + length, nil
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
func fixedSize(typeName string) int {
|
|
389
|
+
switch typeName {
|
|
390
|
+
case "integer", "date":
|
|
391
|
+
return 4
|
|
392
|
+
case "smallint":
|
|
393
|
+
return 2
|
|
394
|
+
case "bigint", "float", "time", "timestamp":
|
|
395
|
+
return 8
|
|
396
|
+
case "boolean":
|
|
397
|
+
return 1
|
|
398
|
+
case "uuid":
|
|
399
|
+
return 16
|
|
400
|
+
default:
|
|
401
|
+
return 0
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
func decodeFixed(raw []byte, typeName string) interface{} {
|
|
406
|
+
switch typeName {
|
|
407
|
+
case "integer":
|
|
408
|
+
return int64(int32(binary.BigEndian.Uint32(raw)))
|
|
409
|
+
case "smallint":
|
|
410
|
+
return int64(int16(binary.BigEndian.Uint16(raw)))
|
|
411
|
+
case "bigint":
|
|
412
|
+
return int64(binary.BigEndian.Uint64(raw))
|
|
413
|
+
case "float":
|
|
414
|
+
return math.Float64frombits(binary.BigEndian.Uint64(raw))
|
|
415
|
+
case "boolean":
|
|
416
|
+
return raw[0] == 1
|
|
417
|
+
case "date":
|
|
418
|
+
return time.Unix(int64(int32(binary.BigEndian.Uint32(raw)))*86400, 0).UTC().Format("2006-01-02")
|
|
419
|
+
case "time":
|
|
420
|
+
total := int64(binary.BigEndian.Uint64(raw))
|
|
421
|
+
seconds, micros := total/1_000_000, total%1_000_000
|
|
422
|
+
value := time.Unix(0, 0).UTC().Add(time.Duration(seconds)*time.Second + time.Duration(micros)*time.Microsecond)
|
|
423
|
+
return value.Format(time.RFC3339Nano)
|
|
424
|
+
case "timestamp":
|
|
425
|
+
return time.Unix(int64(binary.BigEndian.Uint64(raw)), 0).UTC().Format(time.RFC3339Nano)
|
|
426
|
+
case "uuid":
|
|
427
|
+
encoded := hex.EncodeToString(raw)
|
|
428
|
+
return encoded[0:8] + "-" + encoded[8:12] + "-" + encoded[12:16] + "-" + encoded[16:20] + "-" + encoded[20:32]
|
|
429
|
+
default:
|
|
430
|
+
return string(raw)
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
func decodeVariable(raw []byte, typeName string) interface{} {
|
|
435
|
+
if typeName == "blob" {
|
|
436
|
+
return append([]byte(nil), raw...)
|
|
437
|
+
}
|
|
438
|
+
if typeName == "json" {
|
|
439
|
+
var value interface{}
|
|
440
|
+
if json.Unmarshal(raw, &value) == nil {
|
|
441
|
+
return value
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
return string(raw)
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
func bitmapSet(bitmap []byte, index int) bool {
|
|
448
|
+
return len(bitmap) > index/8 && bitmap[index/8]&(1<<uint(index%8)) != 0
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
func matchesFilters(row map[string]interface{}, filters []execution.Filter) bool {
|
|
452
|
+
for _, filter := range filters {
|
|
453
|
+
column := filter.Column
|
|
454
|
+
if dot := strings.LastIndex(column, "."); dot >= 0 {
|
|
455
|
+
column = column[dot+1:]
|
|
456
|
+
}
|
|
457
|
+
if !execution.Matches(row[column], filter.Operator, filter.Value) {
|
|
458
|
+
return false
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
return true
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
func indexedRowIDs(indexes []SnapshotIndex, request SnapshotScanRequest) (map[uint64]struct{}, bool, error) {
|
|
465
|
+
if request.IndexName == "" {
|
|
466
|
+
return nil, false, nil
|
|
467
|
+
}
|
|
468
|
+
var selected *SnapshotIndex
|
|
469
|
+
for index := range indexes {
|
|
470
|
+
if strings.EqualFold(indexes[index].Name, request.IndexName) {
|
|
471
|
+
selected = &indexes[index]
|
|
472
|
+
break
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
if selected == nil || !strings.EqualFold(selected.Type, "btree") {
|
|
476
|
+
return nil, false, nil
|
|
477
|
+
}
|
|
478
|
+
rowIDs := make(map[uint64]struct{})
|
|
479
|
+
matchedFilter := false
|
|
480
|
+
for _, entry := range selected.Entries {
|
|
481
|
+
matches := true
|
|
482
|
+
for index, column := range selected.Columns {
|
|
483
|
+
key, ok := indexKeyPart(entry.Key, index, len(selected.Columns))
|
|
484
|
+
if !ok {
|
|
485
|
+
matches = false
|
|
486
|
+
break
|
|
487
|
+
}
|
|
488
|
+
for _, filter := range request.Filters {
|
|
489
|
+
filterColumn := filter.Column
|
|
490
|
+
if dot := strings.LastIndex(filterColumn, "."); dot >= 0 {
|
|
491
|
+
filterColumn = filterColumn[dot+1:]
|
|
492
|
+
}
|
|
493
|
+
if strings.EqualFold(filterColumn, column) {
|
|
494
|
+
matchedFilter = true
|
|
495
|
+
if !execution.Matches(key, filter.Operator, filter.Value) {
|
|
496
|
+
matches = false
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
if matches {
|
|
502
|
+
rowIDs[entry.RowID] = struct{}{}
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
if !matchedFilter {
|
|
506
|
+
return nil, false, nil
|
|
507
|
+
}
|
|
508
|
+
return rowIDs, true, nil
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
func indexKeyPart(key interface{}, index, count int) (interface{}, bool) {
|
|
512
|
+
if count == 1 {
|
|
513
|
+
return key, true
|
|
514
|
+
}
|
|
515
|
+
parts, ok := key.([]interface{})
|
|
516
|
+
if !ok || index >= len(parts) {
|
|
517
|
+
return nil, false
|
|
518
|
+
}
|
|
519
|
+
return parts[index], true
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
func applyWindow(rows []map[string]interface{}, offset int, limit *int) []map[string]interface{} {
|
|
523
|
+
if offset < 0 {
|
|
524
|
+
offset = 0
|
|
525
|
+
}
|
|
526
|
+
if offset >= len(rows) {
|
|
527
|
+
return []map[string]interface{}{}
|
|
528
|
+
}
|
|
529
|
+
rows = rows[offset:]
|
|
530
|
+
if limit != nil {
|
|
531
|
+
if *limit <= 0 {
|
|
532
|
+
return []map[string]interface{}{}
|
|
533
|
+
}
|
|
534
|
+
if *limit < len(rows) {
|
|
535
|
+
rows = rows[:*limit]
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
return rows
|
|
539
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
package wal
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"bytes"
|
|
5
|
+
"compress/gzip"
|
|
6
|
+
"io"
|
|
7
|
+
)
|
|
8
|
+
|
|
9
|
+
func Gzip(data []byte, level int) ([]byte, error) {
|
|
10
|
+
var output bytes.Buffer
|
|
11
|
+
writer, err := gzip.NewWriterLevel(&output, level)
|
|
12
|
+
if err != nil {
|
|
13
|
+
return nil, err
|
|
14
|
+
}
|
|
15
|
+
if _, err = writer.Write(data); err == nil {
|
|
16
|
+
err = writer.Close()
|
|
17
|
+
}
|
|
18
|
+
if err != nil {
|
|
19
|
+
return nil, err
|
|
20
|
+
}
|
|
21
|
+
return output.Bytes(), nil
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
func Gunzip(data []byte, maxOutput int) ([]byte, error) {
|
|
25
|
+
reader, err := gzip.NewReader(bytes.NewReader(data))
|
|
26
|
+
if err != nil {
|
|
27
|
+
return nil, err
|
|
28
|
+
}
|
|
29
|
+
output, readErr := io.ReadAll(io.LimitReader(reader, int64(maxOutput)+1))
|
|
30
|
+
closeErr := reader.Close()
|
|
31
|
+
if readErr != nil {
|
|
32
|
+
return nil, readErr
|
|
33
|
+
}
|
|
34
|
+
if len(output) > maxOutput {
|
|
35
|
+
return nil, io.ErrShortBuffer
|
|
36
|
+
}
|
|
37
|
+
if closeErr != nil {
|
|
38
|
+
return nil, closeErr
|
|
39
|
+
}
|
|
40
|
+
return output, nil
|
|
41
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
package wal
|
|
2
|
+
|
|
3
|
+
import "sync"
|
|
4
|
+
|
|
5
|
+
// Batch collects approved WAL payloads before Ruby assigns the durable LSN.
|
|
6
|
+
// It does not perform fsync or publish commits; those remain RubyDB rules.
|
|
7
|
+
type Batch struct {
|
|
8
|
+
mu sync.Mutex
|
|
9
|
+
payload [][]byte
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
func (batch *Batch) Add(payload []byte) {
|
|
13
|
+
batch.mu.Lock()
|
|
14
|
+
defer batch.mu.Unlock()
|
|
15
|
+
batch.payload = append(batch.payload, append([]byte(nil), payload...))
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
func (batch *Batch) Drain() [][]byte {
|
|
19
|
+
batch.mu.Lock()
|
|
20
|
+
defer batch.mu.Unlock()
|
|
21
|
+
payload := batch.payload
|
|
22
|
+
batch.payload = nil
|
|
23
|
+
return payload
|
|
24
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
package wal
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"encoding/binary"
|
|
5
|
+
"hash/crc32"
|
|
6
|
+
)
|
|
7
|
+
|
|
8
|
+
// EncodeRecord provides a versioned physical record format for future WAL
|
|
9
|
+
// batching. Ruby remains responsible for LSN allocation and commit ordering.
|
|
10
|
+
// Layout: magic(4), version(1), kind(1), LSN(8), transaction(8), length(4),
|
|
11
|
+
// payload, CRC32(4).
|
|
12
|
+
var recordMagic = [4]byte{'R', 'D', 'W', 'L'}
|
|
13
|
+
|
|
14
|
+
func EncodeRecord(kind byte, lsn, transaction uint64, payload []byte) []byte {
|
|
15
|
+
const headerSize = 26
|
|
16
|
+
output := make([]byte, headerSize+len(payload)+4)
|
|
17
|
+
copy(output[:4], recordMagic[:])
|
|
18
|
+
output[4] = 1
|
|
19
|
+
output[5] = kind
|
|
20
|
+
binary.LittleEndian.PutUint64(output[6:14], lsn)
|
|
21
|
+
binary.LittleEndian.PutUint64(output[14:22], transaction)
|
|
22
|
+
binary.LittleEndian.PutUint32(output[22:26], uint32(len(payload)))
|
|
23
|
+
copy(output[26:], payload)
|
|
24
|
+
checksum := crc32.ChecksumIEEE(output[:26+len(payload)])
|
|
25
|
+
binary.LittleEndian.PutUint32(output[26+len(payload):], checksum)
|
|
26
|
+
return output
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
func VerifyRecord(record []byte) bool {
|
|
30
|
+
if len(record) < 30 || string(record[:4]) != string(recordMagic[:]) || record[4] != 1 {
|
|
31
|
+
return false
|
|
32
|
+
}
|
|
33
|
+
length := int(binary.LittleEndian.Uint32(record[22:26]))
|
|
34
|
+
if length < 0 || len(record) != 30+length {
|
|
35
|
+
return false
|
|
36
|
+
}
|
|
37
|
+
want := binary.LittleEndian.Uint32(record[len(record)-4:])
|
|
38
|
+
got := crc32.ChecksumIEEE(record[:len(record)-4])
|
|
39
|
+
return got == want
|
|
40
|
+
}
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
source "https://rubygems.org"
|
|
4
|
-
|
|
5
|
-
gemspec
|
|
6
|
-
|
|
7
|
-
rails_version = ENV.fetch("RUBYDB_RAILS_VERSION", "7.2")
|
|
8
|
-
gem "activerecord", "~> #{rails_version}.0"
|
|
9
|
-
gem "rake", "~> 13.2"
|
|
10
|
-
gem "rspec", "~> 3.13"
|
|
11
|
-
gem "rubocop", "~> 1.60"
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
source "https://rubygems.org"
|
|
4
|
+
|
|
5
|
+
gemspec
|
|
6
|
+
|
|
7
|
+
rails_version = ENV.fetch("RUBYDB_RAILS_VERSION", "7.2")
|
|
8
|
+
gem "activerecord", "~> #{rails_version}.0"
|
|
9
|
+
gem "rake", "~> 13.2"
|
|
10
|
+
gem "rspec", "~> 3.13"
|
|
11
|
+
gem "rubocop", "~> 1.60"
|