rubydb 0.1.5 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +4 -0
- data/CHANGELOG.md +14 -0
- data/Gemfile.lock +1 -1
- data/README.md +296 -227
- data/Rakefile +6 -1
- data/accelerator/bin/SHA256SUMS +6 -0
- data/accelerator/bin/rubydb-accelerator-darwin-amd64 +0 -0
- data/accelerator/bin/rubydb-accelerator-darwin-arm64 +0 -0
- data/accelerator/bin/rubydb-accelerator-linux-amd64 +0 -0
- data/accelerator/bin/rubydb-accelerator-linux-arm64 +0 -0
- data/accelerator/bin/rubydb-accelerator-windows-amd64.exe +0 -0
- data/accelerator/bin/rubydb-accelerator-windows-arm64.exe +0 -0
- data/accelerator/cmd/rubydb-accelerator/main.go +11 -0
- data/accelerator/go.mod +3 -0
- data/accelerator/internal/execution/aggregate.go +94 -0
- data/accelerator/internal/execution/distinct.go +22 -0
- data/accelerator/internal/execution/filter.go +73 -0
- data/accelerator/internal/execution/join.go +79 -0
- data/accelerator/internal/execution/operators.go +167 -0
- data/accelerator/internal/execution/scan.go +20 -0
- data/accelerator/internal/execution/sort.go +62 -0
- data/accelerator/internal/execution/types.go +136 -0
- data/accelerator/internal/execution/value.go +67 -0
- data/accelerator/internal/memory/arena.go +47 -0
- data/accelerator/internal/memory/reuse.go +22 -0
- data/accelerator/internal/metrics/registry.go +67 -0
- data/accelerator/internal/parallel/bounded_queue.go +56 -0
- data/accelerator/internal/parallel/scheduler.go +47 -0
- data/accelerator/internal/parallel/worker_pool.go +53 -0
- data/accelerator/internal/protocol/cancellation.go +48 -0
- data/accelerator/internal/protocol/columnar.go +263 -0
- data/accelerator/internal/protocol/frame.go +187 -0
- data/accelerator/internal/runtime/worker.go +521 -0
- data/accelerator/internal/storage/page_reader.go +81 -0
- data/accelerator/internal/storage/snapshot_scan.go +539 -0
- data/accelerator/internal/wal/checksum.go +13 -0
- data/accelerator/internal/wal/compression.go +41 -0
- data/accelerator/internal/wal/group_commit.go +24 -0
- data/accelerator/internal/wal/record_encoder.go +40 -0
- data/adapters/activerecord/README.md +8 -3
- data/adapters/activerecord/lib/active_record/connection_adapters/rubydb_adapter.rb +50 -42
- data/adapters/activerecord/rubydb-activerecord.gemspec +1 -1
- data/docs/README.md +3 -1
- data/docs/architecture/go-accelerator.md +179 -0
- data/docs/cli.md +24 -0
- data/docs/contributing/benchmarking.md +16 -0
- data/docs/developer/local-development.md +32 -0
- data/docs/release.md +2 -2
- data/lessons/02-local-development.md +2 -2
- data/lessons/04-rails-complex-apps.md +2 -2
- data/lessons/05-rubydb-production-server.md +2 -2
- data/lessons/07-hybrid-microservices.md +175 -90
- data/lessons/10-release-readiness.md +184 -117
- data/lessons/11-community-adapter.md +323 -0
- data/lessons/12-rails-ecommerce-pressure.md +263 -0
- data/lib/rubydb/accelerator/client.rb +451 -0
- data/lib/rubydb/accelerator/error.rb +22 -0
- data/lib/rubydb/accelerator/manager.rb +606 -0
- data/lib/rubydb/accelerator.rb +13 -0
- data/lib/rubydb/cli/application.rb +6 -1
- data/lib/rubydb/cli/commands/accelerator.rb +72 -0
- data/lib/rubydb/cli/commands/doctor.rb +3 -0
- data/lib/rubydb/client/client.rb +7 -0
- data/lib/rubydb/client/connection.rb +15 -0
- data/lib/rubydb/client/result.rb +5 -1
- data/lib/rubydb/configuration/defaults.rb +12 -0
- data/lib/rubydb/configuration/validation.rb +8 -1
- data/lib/rubydb/execution/accelerator_dispatch.rb +30 -0
- data/lib/rubydb/execution/cost_model.rb +72 -0
- data/lib/rubydb/execution/executor.rb +373 -11
- data/lib/rubydb/execution/operator_selection.rb +57 -0
- data/lib/rubydb/execution/physical_plan.rb +47 -0
- data/lib/rubydb/execution/planner.rb +12 -46
- data/lib/rubydb/execution/sort_executor.rb +22 -8
- data/lib/rubydb/indexes/btree.rb +31 -2
- data/lib/rubydb/rubydb.rb +7 -1
- data/lib/rubydb/server/session.rb +45 -0
- data/lib/rubydb/storage/engine.rb +74 -12
- data/lib/rubydb/storage/snapshot_reader.rb +167 -0
- data/lib/rubydb/version.rb +1 -1
- data/lib/rubydb/wal/archive.rb +17 -0
- data/lib/rubydb/wal/wal.rb +1 -0
- data/rubydb.gemspec +12 -2
- data/scripts/build_accelerator +49 -0
- data/scripts/release +34 -4
- data/scripts/replication_failover_drill +2 -2
- metadata +49 -1
|
@@ -10,6 +10,8 @@ module RubyDB
|
|
|
10
10
|
|
|
11
11
|
def initialize(engine, cte_rows: {}, deadline_at: nil, cancellation: nil)
|
|
12
12
|
@engine = engine
|
|
13
|
+
@accelerator = engine.respond_to?(:accelerator) ? engine.accelerator : nil
|
|
14
|
+
@accelerator_dispatch = AcceleratorDispatch.new(@accelerator)
|
|
13
15
|
@cte_rows = cte_rows
|
|
14
16
|
@deadline_at = deadline_at && (deadline_at.is_a?(Time) ? deadline_at : Time.parse(deadline_at.to_s))
|
|
15
17
|
@cancellation = cancellation
|
|
@@ -21,7 +23,9 @@ module RubyDB
|
|
|
21
23
|
index_scans: 0,
|
|
22
24
|
joins: 0,
|
|
23
25
|
aggregations: 0,
|
|
24
|
-
sorts: 0
|
|
26
|
+
sorts: 0,
|
|
27
|
+
accelerator_requests: 0,
|
|
28
|
+
accelerator_fallbacks: 0
|
|
25
29
|
}
|
|
26
30
|
@lock = Mutex.new
|
|
27
31
|
@current_transaction = nil
|
|
@@ -173,21 +177,36 @@ module RubyDB
|
|
|
173
177
|
end
|
|
174
178
|
|
|
175
179
|
def execute_select(plan)
|
|
180
|
+
@accelerator_window_applied = false
|
|
176
181
|
@stats[:sequential_scans] += 1 if plan.scan_type == :sequential
|
|
177
182
|
@stats[:index_scans] += 1 if plan.scan_type == :index
|
|
178
183
|
|
|
179
184
|
# Qualify source rows before evaluating predicates. This preserves
|
|
180
185
|
# SQL's table/alias namespace while retaining unqualified keys for
|
|
181
186
|
# existing single-table callers.
|
|
182
|
-
|
|
187
|
+
snapshot_rows = accelerate_snapshot_scan(plan)
|
|
188
|
+
rows = snapshot_rows || scan_table(plan)
|
|
189
|
+
accelerated = snapshot_rows || accelerate_scan(rows, plan)
|
|
190
|
+
aggregate_accelerated = false
|
|
191
|
+
unless accelerated
|
|
192
|
+
accelerated_rows = accelerate_aggregate(rows, plan)
|
|
193
|
+
if accelerated_rows
|
|
194
|
+
rows = accelerated_rows
|
|
195
|
+
aggregate_accelerated = true
|
|
196
|
+
end
|
|
197
|
+
end
|
|
198
|
+
if accelerated
|
|
199
|
+
rows = accelerated
|
|
200
|
+
end
|
|
201
|
+
rows = qualify_rows(rows, plan.source_reference || plan.table_name) unless aggregate_accelerated
|
|
183
202
|
plan.joins.each do |join|
|
|
184
203
|
check_deadline!
|
|
185
204
|
right_rows = qualify_rows(scan_table(Plan::Select.new(join[:table].name, [])), join[:table])
|
|
186
|
-
rows = execute_join(rows, right_rows, join)
|
|
205
|
+
rows = accelerate_join(rows, right_rows, join) || execute_join(rows, right_rows, join)
|
|
187
206
|
end
|
|
188
207
|
|
|
189
208
|
# Apply filters (WHERE clause)
|
|
190
|
-
if plan.predicate
|
|
209
|
+
if plan.predicate && !accelerated && !aggregate_accelerated
|
|
191
210
|
rows = rows.each_with_index.filter_map do |row, index|
|
|
192
211
|
check_deadline! if (index & 255).zero?
|
|
193
212
|
row if evaluate_predicate(plan.predicate, row)
|
|
@@ -199,14 +218,14 @@ module RubyDB
|
|
|
199
218
|
# ORDER BY is evaluated against the source rows, so a column used
|
|
200
219
|
# solely for ordering remains available even when it is not selected.
|
|
201
220
|
aggregating = plan.aggregates && !plan.aggregates.empty?
|
|
202
|
-
if plan.order_by && !plan.order_by.empty? && !aggregating
|
|
221
|
+
if plan.order_by && !plan.order_by.empty? && !aggregating && !accelerated
|
|
203
222
|
rows = sort_rows(rows, plan.order_by)
|
|
204
223
|
end
|
|
205
224
|
|
|
206
|
-
if aggregating
|
|
225
|
+
if aggregating && !aggregate_accelerated
|
|
207
226
|
rows = aggregate_rows(rows, plan.group_by || [], plan.projections)
|
|
208
227
|
rows = rows.select { |row| evaluate_predicate(plan.having, row) } if plan.having
|
|
209
|
-
elsif plan.projections
|
|
228
|
+
elsif !aggregating && plan.projections
|
|
210
229
|
# Apply projections (SELECT columns)
|
|
211
230
|
rows = rows.each_with_index.map do |row, index|
|
|
212
231
|
check_deadline! if (index & 255).zero?
|
|
@@ -226,7 +245,7 @@ module RubyDB
|
|
|
226
245
|
end
|
|
227
246
|
|
|
228
247
|
# Apply LIMIT and OFFSET
|
|
229
|
-
if plan.limit
|
|
248
|
+
if plan.limit && !@accelerator_window_applied
|
|
230
249
|
offset = plan.offset || 0
|
|
231
250
|
rows = rows[offset, plan.limit] || []
|
|
232
251
|
elsif plan.offset
|
|
@@ -576,6 +595,338 @@ module RubyDB
|
|
|
576
595
|
end
|
|
577
596
|
|
|
578
597
|
# Scan operations
|
|
598
|
+
def accelerate_snapshot_scan(plan)
|
|
599
|
+
return nil unless @accelerator
|
|
600
|
+
return nil unless @accelerator.read_pipeline?
|
|
601
|
+
return nil unless @accelerator_dispatch.preferred?(:snapshot_scan)
|
|
602
|
+
return nil unless plan.table_name && plan.joins.empty?
|
|
603
|
+
return nil if @cte_rows.any?
|
|
604
|
+
return nil if plan.distinct || plan.having || (plan.aggregates && !plan.aggregates.empty?)
|
|
605
|
+
return nil if plan.estimated_rows.to_i < @accelerator.min_rows
|
|
606
|
+
|
|
607
|
+
filters = accelerator_filters(plan.predicate)
|
|
608
|
+
order_by = accelerator_order_by(plan.order_by)
|
|
609
|
+
return nil if filters.nil? || order_by.nil?
|
|
610
|
+
|
|
611
|
+
columns = @engine.table_columns(plan.table_name)
|
|
612
|
+
column_names = columns.map { |column| column.name.to_s }
|
|
613
|
+
column_types = columns.each_with_object({}) { |column, types| types[column.name.to_s] = column.type_class.to_s }
|
|
614
|
+
index_name = if plan.scan_type == :index && plan.index&.type&.to_sym == :btree
|
|
615
|
+
plan.index.name.to_s
|
|
616
|
+
end
|
|
617
|
+
storage_order = order_by.map { |order| order.merge(column: order[:column].to_s.split(".").last) }
|
|
618
|
+
windowed_projection = Array(plan.projections).any? do |projection|
|
|
619
|
+
expression = projection.respond_to?(:expression) ? projection.expression : projection
|
|
620
|
+
expression.is_a?(SQL::AST::FunctionCall) && expression.window
|
|
621
|
+
end
|
|
622
|
+
window_limit = if !windowed_projection && !plan.distinct && !(plan.aggregates && !plan.aggregates.empty?)
|
|
623
|
+
plan.limit
|
|
624
|
+
end
|
|
625
|
+
window_eligible = !windowed_projection && !plan.distinct && !(plan.aggregates && !plan.aggregates.empty?)
|
|
626
|
+
window_offset = (window_limit || window_eligible) ? (plan.offset || 0) : 0
|
|
627
|
+
|
|
628
|
+
@stats[:accelerator_requests] += 1
|
|
629
|
+
calibrating = !@accelerator.manager.performance_calibrated?(:snapshot_scan)
|
|
630
|
+
started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
631
|
+
result = nil
|
|
632
|
+
@engine.with_accelerator_snapshot do |manifest|
|
|
633
|
+
result = @accelerator.snapshot_scan(
|
|
634
|
+
manifest,
|
|
635
|
+
table: plan.table_name,
|
|
636
|
+
columns: column_names,
|
|
637
|
+
column_types: column_types,
|
|
638
|
+
filters: filters,
|
|
639
|
+
order_by: storage_order,
|
|
640
|
+
index_name: index_name,
|
|
641
|
+
limit: window_limit,
|
|
642
|
+
offset: window_offset,
|
|
643
|
+
batch_size: window_limit ? [window_limit, 1024].min : 1024
|
|
644
|
+
)
|
|
645
|
+
end
|
|
646
|
+
return nil unless result
|
|
647
|
+
@accelerator_window_applied = !window_limit.nil? || window_offset.positive?
|
|
648
|
+
return result[:rows] unless calibrating
|
|
649
|
+
|
|
650
|
+
ruby_started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
651
|
+
ruby_rows = ruby_scan_rows(@engine.select_rows(plan.table_name, columns), plan, apply_window: !window_limit.nil?)
|
|
652
|
+
go_ms = elapsed_milliseconds(started_at)
|
|
653
|
+
ruby_ms = elapsed_milliseconds(ruby_started_at)
|
|
654
|
+
unless accelerator_results_equivalent?(result[:rows], ruby_rows)
|
|
655
|
+
@stats[:accelerator_fallbacks] += 1
|
|
656
|
+
@accelerator.manager.record_performance(:snapshot_scan, ruby_ms: ruby_ms, go_ms: Float::INFINITY)
|
|
657
|
+
@accelerator.manager.record_performance(:scan, ruby_ms: ruby_ms, go_ms: Float::INFINITY)
|
|
658
|
+
raise ExecutionError, "RubyDB accelerator returned a different snapshot scan result" if @accelerator.manager.mode == "required"
|
|
659
|
+
|
|
660
|
+
return ruby_rows
|
|
661
|
+
end
|
|
662
|
+
@accelerator.manager.record_performance(:snapshot_scan, ruby_ms: ruby_ms, go_ms: go_ms)
|
|
663
|
+
@accelerator.manager.record_performance(:scan, ruby_ms: ruby_ms, go_ms: go_ms)
|
|
664
|
+
@accelerator.manager.acceleration_preferred?(:snapshot_scan) ? result[:rows] : ruby_rows
|
|
665
|
+
rescue RubyDB::Accelerator::Error, RubyDB::StorageError
|
|
666
|
+
@stats[:accelerator_fallbacks] += 1
|
|
667
|
+
raise if @accelerator.manager.mode == "required"
|
|
668
|
+
|
|
669
|
+
nil
|
|
670
|
+
end
|
|
671
|
+
|
|
672
|
+
def accelerate_scan(rows, plan)
|
|
673
|
+
return nil unless @accelerator
|
|
674
|
+
return nil unless @accelerator.read_pipeline?
|
|
675
|
+
return nil if rows.size < @accelerator.min_rows
|
|
676
|
+
return nil unless @accelerator_dispatch.preferred?(:scan)
|
|
677
|
+
return nil unless plan.joins.empty?
|
|
678
|
+
return nil if plan.distinct || (plan.aggregates && !plan.aggregates.empty?)
|
|
679
|
+
|
|
680
|
+
filters = accelerator_filters(plan.predicate)
|
|
681
|
+
order_by = accelerator_order_by(plan.order_by)
|
|
682
|
+
return nil if filters.nil? || order_by.nil?
|
|
683
|
+
return nil if filters.empty? && order_by.empty?
|
|
684
|
+
|
|
685
|
+
calibrating = !@accelerator.manager.performance_calibrated?(:scan)
|
|
686
|
+
@stats[:accelerator_requests] += 1
|
|
687
|
+
started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
688
|
+
result = @accelerator.rows_pipeline(
|
|
689
|
+
rows,
|
|
690
|
+
filters: filters,
|
|
691
|
+
order_by: order_by,
|
|
692
|
+
# Ruby applies LIMIT/OFFSET after projection and aggregation. Do
|
|
693
|
+
# not apply them in Go here or the final Ruby stage would paginate
|
|
694
|
+
# the result twice.
|
|
695
|
+
limit: nil,
|
|
696
|
+
offset: 0
|
|
697
|
+
)
|
|
698
|
+
return nil unless result
|
|
699
|
+
return result[:rows] unless calibrating
|
|
700
|
+
|
|
701
|
+
ruby_started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
702
|
+
ruby_rows = ruby_scan_rows(rows, plan)
|
|
703
|
+
go_ms = elapsed_milliseconds(started_at)
|
|
704
|
+
ruby_ms = elapsed_milliseconds(ruby_started_at)
|
|
705
|
+
if !accelerator_results_equivalent?(result[:rows], ruby_rows)
|
|
706
|
+
@stats[:accelerator_fallbacks] += 1
|
|
707
|
+
@accelerator.manager.record_performance(:scan, ruby_ms: ruby_ms, go_ms: Float::INFINITY)
|
|
708
|
+
if @accelerator.manager.mode == "required"
|
|
709
|
+
raise ExecutionError, "RubyDB accelerator returned a different scan result"
|
|
710
|
+
end
|
|
711
|
+
|
|
712
|
+
return ruby_rows
|
|
713
|
+
end
|
|
714
|
+
@accelerator.manager.record_performance(:scan, ruby_ms: ruby_ms, go_ms: go_ms)
|
|
715
|
+
@accelerator.manager.acceleration_preferred?(:scan) ? result[:rows] : ruby_rows
|
|
716
|
+
rescue RubyDB::Accelerator::Error
|
|
717
|
+
@stats[:accelerator_fallbacks] += 1
|
|
718
|
+
raise if @accelerator.manager.mode == "required"
|
|
719
|
+
|
|
720
|
+
nil
|
|
721
|
+
end
|
|
722
|
+
|
|
723
|
+
def accelerate_aggregate(rows, plan)
|
|
724
|
+
return nil unless @accelerator
|
|
725
|
+
return nil unless @accelerator.read_pipeline?
|
|
726
|
+
return nil if rows.size < @accelerator.min_rows
|
|
727
|
+
return nil unless @accelerator_dispatch.preferred?(:aggregate)
|
|
728
|
+
return nil unless plan.joins.empty?
|
|
729
|
+
return nil if plan.distinct || plan.having || plan.order_by&.any?
|
|
730
|
+
return nil unless plan.aggregates && !plan.aggregates.empty?
|
|
731
|
+
|
|
732
|
+
filters = accelerator_filters(plan.predicate)
|
|
733
|
+
group_by = Array(plan.group_by).map { |expression| accelerator_identifier(expression) }
|
|
734
|
+
return nil if filters.nil? || group_by.any?(&:nil?)
|
|
735
|
+
|
|
736
|
+
definitions = []
|
|
737
|
+
projections = Array(plan.projections)
|
|
738
|
+
projections.each do |projection|
|
|
739
|
+
expression = projection.respond_to?(:expression) ? projection.expression : projection
|
|
740
|
+
if aggregate_function?(expression)
|
|
741
|
+
return nil if expression.distinct || expression.arguments.length > 1
|
|
742
|
+
|
|
743
|
+
argument = expression.arguments.first
|
|
744
|
+
column = argument.is_a?(SQL::AST::Star) ? "*" : accelerator_identifier(argument)
|
|
745
|
+
return nil unless column
|
|
746
|
+
alias_name = projection.respond_to?(:alias_name) ? projection.alias_name : nil
|
|
747
|
+
return nil if alias_name.nil? || alias_name.to_s.empty?
|
|
748
|
+
|
|
749
|
+
definitions << {function: expression.name.to_s.upcase, column: column, alias: alias_name.to_s}
|
|
750
|
+
else
|
|
751
|
+
column = accelerator_identifier(expression)
|
|
752
|
+
return nil unless column && group_by.include?(column)
|
|
753
|
+
end
|
|
754
|
+
end
|
|
755
|
+
|
|
756
|
+
return nil if definitions.empty?
|
|
757
|
+
|
|
758
|
+
calibrating = !@accelerator.manager.performance_calibrated?(:aggregate)
|
|
759
|
+
@stats[:accelerator_requests] += 1
|
|
760
|
+
started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
761
|
+
result = @accelerator.rows_pipeline(
|
|
762
|
+
rows,
|
|
763
|
+
filters: filters,
|
|
764
|
+
group_by: group_by,
|
|
765
|
+
aggregates: definitions,
|
|
766
|
+
limit: nil,
|
|
767
|
+
offset: 0
|
|
768
|
+
)
|
|
769
|
+
accelerated_aggregate_rows = result && result[:aggregates]
|
|
770
|
+
return nil unless accelerated_aggregate_rows
|
|
771
|
+
|
|
772
|
+
projected_rows = accelerated_aggregate_rows.map do |row|
|
|
773
|
+
projections.each_with_object({}) do |projection, projected|
|
|
774
|
+
name = projection_name(projection)
|
|
775
|
+
projected[name] = row[name]
|
|
776
|
+
end
|
|
777
|
+
end
|
|
778
|
+
return projected_rows unless calibrating
|
|
779
|
+
|
|
780
|
+
ruby_started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
781
|
+
filtered_rows = plan.predicate ? rows.select { |row| evaluate_predicate(plan.predicate, row) } : rows
|
|
782
|
+
ruby_rows = aggregate_rows(filtered_rows, plan.group_by || [], projections)
|
|
783
|
+
go_ms = elapsed_milliseconds(started_at)
|
|
784
|
+
ruby_ms = elapsed_milliseconds(ruby_started_at)
|
|
785
|
+
if !accelerator_results_equivalent?(projected_rows, ruby_rows)
|
|
786
|
+
@stats[:accelerator_fallbacks] += 1
|
|
787
|
+
@accelerator.manager.record_performance(:aggregate, ruby_ms: ruby_ms, go_ms: Float::INFINITY)
|
|
788
|
+
raise ExecutionError, "RubyDB accelerator returned a different aggregate result" if @accelerator.manager.mode == "required"
|
|
789
|
+
|
|
790
|
+
return ruby_rows
|
|
791
|
+
end
|
|
792
|
+
@accelerator.manager.record_performance(:aggregate, ruby_ms: ruby_ms, go_ms: go_ms)
|
|
793
|
+
@accelerator.manager.acceleration_preferred?(:aggregate) ? projected_rows : ruby_rows
|
|
794
|
+
rescue RubyDB::Accelerator::Error
|
|
795
|
+
@stats[:accelerator_fallbacks] += 1
|
|
796
|
+
raise if @accelerator.manager.mode == "required"
|
|
797
|
+
|
|
798
|
+
nil
|
|
799
|
+
end
|
|
800
|
+
|
|
801
|
+
def ruby_scan_rows(rows, plan, apply_window: false)
|
|
802
|
+
source_rows = rows.map { |row| row.each_with_object({}) { |(key, value), copy| copy[key.to_s] = value } }
|
|
803
|
+
result = plan.predicate ? source_rows.select { |row| evaluate_predicate(plan.predicate, row) } : source_rows
|
|
804
|
+
order_by = accelerator_order_by(plan.order_by)
|
|
805
|
+
result = sort_rows(result, order_by) if order_by && !order_by.empty?
|
|
806
|
+
if apply_window
|
|
807
|
+
offset = plan.offset || 0
|
|
808
|
+
result = result[offset, plan.limit] || [] if plan.limit
|
|
809
|
+
result = result[offset..] || [] if !plan.limit && offset.positive?
|
|
810
|
+
end
|
|
811
|
+
result
|
|
812
|
+
end
|
|
813
|
+
|
|
814
|
+
def accelerate_join(left_rows, right_rows, join)
|
|
815
|
+
return nil unless @accelerator
|
|
816
|
+
return nil unless @accelerator.read_pipeline?
|
|
817
|
+
return nil if left_rows.size + right_rows.size < @accelerator.min_rows
|
|
818
|
+
return nil unless join[:type].to_sym == :inner
|
|
819
|
+
|
|
820
|
+
predicate = join[:predicate]
|
|
821
|
+
return nil unless predicate.is_a?(Predicate::Comparison) && predicate.operator.to_s.casecmp?("eq")
|
|
822
|
+
|
|
823
|
+
left_key = accelerator_identifier(predicate.left)
|
|
824
|
+
right_key = accelerator_identifier(predicate.right)
|
|
825
|
+
return nil unless left_key && right_key
|
|
826
|
+
return nil unless left_rows.first&.key?(left_key) && right_rows.first&.key?(right_key)
|
|
827
|
+
return nil unless @accelerator_dispatch.preferred?(:join)
|
|
828
|
+
|
|
829
|
+
calibrating = !@accelerator.manager.performance_calibrated?(:join)
|
|
830
|
+
@stats[:accelerator_requests] += 1
|
|
831
|
+
started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
832
|
+
result = @accelerator.hash_join(left_rows, right_rows, left_key: left_key, right_key: right_key)
|
|
833
|
+
return nil unless result
|
|
834
|
+
return result[:rows] unless calibrating
|
|
835
|
+
|
|
836
|
+
ruby_started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
837
|
+
ruby_rows = execute_join(left_rows, right_rows, join)
|
|
838
|
+
go_ms = elapsed_milliseconds(started_at)
|
|
839
|
+
ruby_ms = elapsed_milliseconds(ruby_started_at)
|
|
840
|
+
unless accelerator_results_equivalent?(result[:rows], ruby_rows)
|
|
841
|
+
@stats[:accelerator_fallbacks] += 1
|
|
842
|
+
@accelerator.manager.record_performance(:join, ruby_ms: ruby_ms, go_ms: Float::INFINITY)
|
|
843
|
+
raise ExecutionError, "RubyDB accelerator returned a different join result" if @accelerator.manager.mode == "required"
|
|
844
|
+
|
|
845
|
+
return ruby_rows
|
|
846
|
+
end
|
|
847
|
+
@accelerator.manager.record_performance(:join, ruby_ms: ruby_ms, go_ms: go_ms)
|
|
848
|
+
@accelerator.manager.acceleration_preferred?(:join) ? result[:rows] : ruby_rows
|
|
849
|
+
rescue RubyDB::Accelerator::Error
|
|
850
|
+
@stats[:accelerator_fallbacks] += 1
|
|
851
|
+
raise if @accelerator.manager.mode == "required"
|
|
852
|
+
|
|
853
|
+
nil
|
|
854
|
+
end
|
|
855
|
+
|
|
856
|
+
def elapsed_milliseconds(started_at)
|
|
857
|
+
(Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at) * 1000.0
|
|
858
|
+
end
|
|
859
|
+
|
|
860
|
+
def accelerator_results_equivalent?(left, right)
|
|
861
|
+
@accelerator_dispatch.equivalent?(left, right)
|
|
862
|
+
end
|
|
863
|
+
|
|
864
|
+
def accelerator_filters(predicate)
|
|
865
|
+
return [] unless predicate
|
|
866
|
+
|
|
867
|
+
case predicate
|
|
868
|
+
when Predicate::And
|
|
869
|
+
left = accelerator_filters(predicate.left)
|
|
870
|
+
right = accelerator_filters(predicate.right)
|
|
871
|
+
return nil if left.nil? || right.nil?
|
|
872
|
+
|
|
873
|
+
left + right
|
|
874
|
+
when Predicate::Comparison
|
|
875
|
+
column = accelerator_identifier(predicate.left)
|
|
876
|
+
value = accelerator_literal(predicate.right)
|
|
877
|
+
return nil if column.nil? || value.equal?(:unsupported)
|
|
878
|
+
|
|
879
|
+
[{column: column, operator: predicate.operator.to_s, value: value}]
|
|
880
|
+
when Predicate::Between
|
|
881
|
+
column = accelerator_identifier(predicate.expression)
|
|
882
|
+
low = accelerator_literal(predicate.low)
|
|
883
|
+
high = accelerator_literal(predicate.high)
|
|
884
|
+
return nil if column.nil? || low.equal?(:unsupported) || high.equal?(:unsupported)
|
|
885
|
+
|
|
886
|
+
[
|
|
887
|
+
{column: column, operator: "gte", value: low},
|
|
888
|
+
{column: column, operator: "lte", value: high}
|
|
889
|
+
]
|
|
890
|
+
when Predicate::IsNull
|
|
891
|
+
column = accelerator_identifier(predicate.expression)
|
|
892
|
+
return nil unless column
|
|
893
|
+
|
|
894
|
+
[{column: column, operator: predicate.negated ? "is_not_null" : "is_null"}]
|
|
895
|
+
when Predicate::Like
|
|
896
|
+
column = accelerator_identifier(predicate.expression)
|
|
897
|
+
value = accelerator_literal(predicate.pattern)
|
|
898
|
+
return nil if column.nil? || value.equal?(:unsupported)
|
|
899
|
+
|
|
900
|
+
[{column: column, operator: "like", value: value}]
|
|
901
|
+
end
|
|
902
|
+
end
|
|
903
|
+
|
|
904
|
+
def accelerator_order_by(order_by)
|
|
905
|
+
order_by.map do |order|
|
|
906
|
+
expression = order.is_a?(Hash) ? order[:column] : order.expression
|
|
907
|
+
column = accelerator_identifier(expression)
|
|
908
|
+
return nil unless column
|
|
909
|
+
|
|
910
|
+
direction = order.is_a?(Hash) ? order[:direction] : order.direction
|
|
911
|
+
{column: column, direction: direction.to_s}
|
|
912
|
+
end
|
|
913
|
+
end
|
|
914
|
+
|
|
915
|
+
def accelerator_identifier(expression)
|
|
916
|
+
return expression.to_s if expression.is_a?(String) || expression.is_a?(Symbol)
|
|
917
|
+
return expression.name.to_s if expression.is_a?(Expression::Column)
|
|
918
|
+
return nil unless expression.is_a?(SQL::AST::Identifier)
|
|
919
|
+
|
|
920
|
+
expression.name.to_s
|
|
921
|
+
end
|
|
922
|
+
|
|
923
|
+
def accelerator_literal(expression)
|
|
924
|
+
return expression.value if expression.is_a?(Expression::Literal)
|
|
925
|
+
return :unsupported unless expression.is_a?(SQL::AST::Literal)
|
|
926
|
+
|
|
927
|
+
expression.value
|
|
928
|
+
end
|
|
929
|
+
|
|
579
930
|
def scan_table(plan)
|
|
580
931
|
table_name = plan.table_name
|
|
581
932
|
return [{}] unless table_name
|
|
@@ -1112,8 +1463,8 @@ module RubyDB
|
|
|
1112
1463
|
comparison = 0
|
|
1113
1464
|
order_by.each do |order|
|
|
1114
1465
|
col = (order.is_a?(Hash) ? order[:column] : order.column).to_s
|
|
1115
|
-
val_a = a
|
|
1116
|
-
val_b = b
|
|
1466
|
+
val_a = order_value(a, col)
|
|
1467
|
+
val_b = order_value(b, col)
|
|
1117
1468
|
|
|
1118
1469
|
comparison = if val_a.nil? && val_b.nil?
|
|
1119
1470
|
0
|
|
@@ -1126,13 +1477,24 @@ module RubyDB
|
|
|
1126
1477
|
end
|
|
1127
1478
|
|
|
1128
1479
|
direction = order.is_a?(Hash) ? order[:direction] : order.direction
|
|
1129
|
-
comparison = -comparison if direction
|
|
1480
|
+
comparison = -comparison if direction.to_s.casecmp?("desc")
|
|
1130
1481
|
break unless comparison == 0
|
|
1131
1482
|
end
|
|
1132
1483
|
comparison
|
|
1133
1484
|
end
|
|
1134
1485
|
end
|
|
1135
1486
|
|
|
1487
|
+
def order_value(row, column)
|
|
1488
|
+
return row[column] if row.key?(column)
|
|
1489
|
+
return row[column.to_sym] if row.key?(column.to_sym)
|
|
1490
|
+
|
|
1491
|
+
short_column = column.split(".").last
|
|
1492
|
+
return row[short_column] if row.key?(short_column)
|
|
1493
|
+
return row[short_column.to_sym] if row.key?(short_column.to_sym)
|
|
1494
|
+
|
|
1495
|
+
nil
|
|
1496
|
+
end
|
|
1497
|
+
|
|
1136
1498
|
def get_column_names(plan)
|
|
1137
1499
|
if plan.projections
|
|
1138
1500
|
plan.projections.map { |projection| projection_name(projection) }
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RubyDB
|
|
4
|
+
module Execution
|
|
5
|
+
class OperatorSelection
|
|
6
|
+
def initialize(engine)
|
|
7
|
+
@engine = engine
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def choose_scan(plan)
|
|
11
|
+
return plan unless plan.type == :select
|
|
12
|
+
|
|
13
|
+
table_name = plan.table_name
|
|
14
|
+
return plan.set_scan_type(:sequential) unless table_name
|
|
15
|
+
|
|
16
|
+
index = matching_index(plan)
|
|
17
|
+
index ? plan.set_scan_type(:index, index) : plan.set_scan_type(:sequential)
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def physical_operator(plan)
|
|
21
|
+
return :index_scan if plan.scan_type == :index
|
|
22
|
+
return :sequential_scan if plan.scan_type == :sequential
|
|
23
|
+
return :hash_join if plan.joins&.any?
|
|
24
|
+
|
|
25
|
+
plan.type
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
private
|
|
29
|
+
|
|
30
|
+
def matching_index(plan)
|
|
31
|
+
return nil unless @engine.respond_to?(:index_manager) && plan.predicate
|
|
32
|
+
|
|
33
|
+
columns = predicate_columns(plan.predicate).map(&:to_s)
|
|
34
|
+
@engine.index_manager.get_indexes_for_table(plan.table_name).find do |index|
|
|
35
|
+
index.columns.any? { |column| columns.include?(column.to_s) }
|
|
36
|
+
end
|
|
37
|
+
rescue
|
|
38
|
+
nil
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def predicate_columns(predicate)
|
|
42
|
+
case predicate
|
|
43
|
+
when Predicate::Comparison
|
|
44
|
+
[predicate.left.respond_to?(:name) ? predicate.left.name : nil].compact
|
|
45
|
+
when Predicate::And, Predicate::Or
|
|
46
|
+
predicate_columns(predicate.left) + predicate_columns(predicate.right)
|
|
47
|
+
when Predicate::Not
|
|
48
|
+
predicate_columns(predicate.operand)
|
|
49
|
+
when Predicate::Between, Predicate::In, Predicate::IsNull, Predicate::Like
|
|
50
|
+
[predicate.expression.respond_to?(:name) ? predicate.expression.name : nil].compact
|
|
51
|
+
else
|
|
52
|
+
[]
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module RubyDB
|
|
4
|
+
module Execution
|
|
5
|
+
# A compact description of the physical choices made for a logical Plan.
|
|
6
|
+
# Plan remains the executable object; this value is for explainability,
|
|
7
|
+
# cache keys, and the Ruby/Go dispatch boundary.
|
|
8
|
+
class PhysicalPlan
|
|
9
|
+
attr_reader :logical_plan, :operator, :estimated_cost, :estimated_rows, :properties
|
|
10
|
+
|
|
11
|
+
def self.from(plan, operator: nil, estimated_cost: nil, estimated_rows: nil, properties: {})
|
|
12
|
+
new(
|
|
13
|
+
plan,
|
|
14
|
+
operator: operator || default_operator(plan),
|
|
15
|
+
estimated_cost: estimated_cost.nil? ? plan.estimated_cost : estimated_cost,
|
|
16
|
+
estimated_rows: estimated_rows.nil? ? plan.estimated_rows : estimated_rows,
|
|
17
|
+
properties: properties
|
|
18
|
+
)
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def self.default_operator(plan)
|
|
22
|
+
return :index_scan if plan.respond_to?(:scan_type) && plan.scan_type == :index
|
|
23
|
+
return :sequential_scan if plan.respond_to?(:scan_type) && plan.scan_type == :sequential
|
|
24
|
+
|
|
25
|
+
plan.type
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def initialize(logical_plan, operator:, estimated_cost:, estimated_rows:, properties:)
|
|
29
|
+
@logical_plan = logical_plan
|
|
30
|
+
@operator = operator.to_sym
|
|
31
|
+
@estimated_cost = estimated_cost
|
|
32
|
+
@estimated_rows = estimated_rows
|
|
33
|
+
@properties = properties.freeze
|
|
34
|
+
freeze
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def to_h
|
|
38
|
+
{
|
|
39
|
+
operator: @operator,
|
|
40
|
+
estimated_cost: @estimated_cost,
|
|
41
|
+
estimated_rows: @estimated_rows,
|
|
42
|
+
properties: @properties
|
|
43
|
+
}
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
@@ -9,6 +9,8 @@ module RubyDB
|
|
|
9
9
|
def initialize(engine)
|
|
10
10
|
@engine = engine
|
|
11
11
|
@catalog = engine.catalog if engine.respond_to?(:catalog)
|
|
12
|
+
@cost_model = CostModel.new(engine)
|
|
13
|
+
@operator_selection = OperatorSelection.new(engine)
|
|
12
14
|
@stats = {
|
|
13
15
|
plans_created: 0,
|
|
14
16
|
optimizations_applied: 0
|
|
@@ -327,31 +329,7 @@ module RubyDB
|
|
|
327
329
|
|
|
328
330
|
# Scan method selection
|
|
329
331
|
def choose_scan_method(plan)
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
table_name = plan.table_name
|
|
333
|
-
return plan.set_scan_type(:sequential) unless table_name
|
|
334
|
-
|
|
335
|
-
# Check if there's an index that can be used
|
|
336
|
-
if @engine.respond_to?(:index_manager)
|
|
337
|
-
indexes = @engine.index_manager.get_indexes_for_table(table_name)
|
|
338
|
-
|
|
339
|
-
# Try to find a matching index for the predicate
|
|
340
|
-
if plan.predicate && indexes.any?
|
|
341
|
-
index = find_matching_index(plan.predicate, indexes)
|
|
342
|
-
if index
|
|
343
|
-
plan.set_scan_type(:index, index)
|
|
344
|
-
return
|
|
345
|
-
end
|
|
346
|
-
end
|
|
347
|
-
|
|
348
|
-
# A full index scan is not a substitute for a table scan: it can
|
|
349
|
-
# omit rows while an index is being maintained or rebuilt. Only use
|
|
350
|
-
# an index when the predicate specifically matches its columns.
|
|
351
|
-
end
|
|
352
|
-
|
|
353
|
-
# Default to sequential scan
|
|
354
|
-
plan.set_scan_type(:sequential)
|
|
332
|
+
@operator_selection.choose_scan(plan)
|
|
355
333
|
end
|
|
356
334
|
|
|
357
335
|
def find_matching_index(predicate, indexes)
|
|
@@ -389,28 +367,16 @@ module RubyDB
|
|
|
389
367
|
|
|
390
368
|
# Cost estimation
|
|
391
369
|
def estimate_cost(plan)
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
@engine.table_row_count(table_name)
|
|
395
|
-
rescue
|
|
396
|
-
0
|
|
397
|
-
end
|
|
370
|
+
@cost_model.apply(plan)
|
|
371
|
+
end
|
|
398
372
|
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
plan
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
estimated_rows = (row_count * selectivity).ceil
|
|
407
|
-
plan.set_cost(estimated_rows * 0.1, estimated_rows)
|
|
408
|
-
else
|
|
409
|
-
plan.set_cost(row_count * 0.5, row_count)
|
|
410
|
-
end
|
|
411
|
-
else
|
|
412
|
-
plan.set_cost(row_count, row_count)
|
|
413
|
-
end
|
|
373
|
+
def physical_plan(plan)
|
|
374
|
+
PhysicalPlan.from(
|
|
375
|
+
plan,
|
|
376
|
+
operator: @operator_selection.physical_operator(plan),
|
|
377
|
+
estimated_cost: plan.estimated_cost,
|
|
378
|
+
estimated_rows: plan.estimated_rows
|
|
379
|
+
)
|
|
414
380
|
end
|
|
415
381
|
|
|
416
382
|
def estimate_selectivity(predicate)
|