duckdb 0.3.2-dev1000.0 → 0.3.2-dev1051.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/duckdb.cpp CHANGED
@@ -24867,8 +24867,9 @@ public:
24867
24867
  void PrepareMergePhase();
24868
24868
  //! Initializes the global sort state for another round of merging
24869
24869
  void InitializeMergeRound();
24870
- //! Completes the cascaded merge sort round
24871
- void CompleteMergeRound();
24870
+ //! Completes the cascaded merge sort round.
24871
+ //! Pass true if you wish to use the radix data for further comparisons.
24872
+ void CompleteMergeRound(bool keep_radix_data = false);
24872
24873
 
24873
24874
  public:
24874
24875
  //! The lock for updating the order global state
@@ -26764,7 +26765,7 @@ void GlobalSortState::InitializeMergeRound() {
26764
26765
  }
26765
26766
  }
26766
26767
 
26767
- void GlobalSortState::CompleteMergeRound() {
26768
+ void GlobalSortState::CompleteMergeRound(bool keep_radix_data) {
26768
26769
  sorted_blocks.clear();
26769
26770
  for (auto &sorted_block_vector : sorted_blocks_temp) {
26770
26771
  sorted_blocks.push_back(make_unique<SortedBlock>(buffer_manager, *this));
@@ -26776,7 +26777,7 @@ void GlobalSortState::CompleteMergeRound() {
26776
26777
  odd_one_out = nullptr;
26777
26778
  }
26778
26779
  // Only one block left: Done!
26779
- if (sorted_blocks.size() == 1) {
26780
+ if (sorted_blocks.size() == 1 && !keep_radix_data) {
26780
26781
  sorted_blocks[0]->radix_sorting_data.clear();
26781
26782
  sorted_blocks[0]->blob_sorting_data = nullptr;
26782
26783
  }
@@ -27075,6 +27076,7 @@ void PayloadScanner::Scan(DataChunk &chunk) {
27075
27076
  auto count = MinValue((idx_t)STANDARD_VECTOR_SIZE, total_count - total_scanned);
27076
27077
  if (count == 0) {
27077
27078
  D_ASSERT(read_state.block_idx == sorted_data.data_blocks.size());
27079
+ chunk.SetCardinality(count);
27078
27080
  return;
27079
27081
  }
27080
27082
  // Eagerly delete references to blocks that we've passed
@@ -47663,366 +47665,6 @@ idx_t JoinHashTable::FillWithHTOffsets(data_ptr_t *key_locations, JoinHTScanStat
47663
47665
  }
47664
47666
  return key_count;
47665
47667
  }
47666
- } // namespace duckdb
47667
- //===----------------------------------------------------------------------===//
47668
- // DuckDB
47669
- //
47670
- // duckdb/execution/merge_join.hpp
47671
- //
47672
- //
47673
- //===----------------------------------------------------------------------===//
47674
-
47675
-
47676
-
47677
-
47678
-
47679
-
47680
-
47681
-
47682
- namespace duckdb {
47683
-
47684
- struct MergeOrder {
47685
- SelectionVector order;
47686
- idx_t count;
47687
- VectorData vdata;
47688
- };
47689
-
47690
- enum MergeInfoType : uint8_t { SCALAR_MERGE_INFO = 1, CHUNK_MERGE_INFO = 2 };
47691
-
47692
- struct MergeInfo {
47693
- MergeInfo(MergeInfoType info_type, LogicalType type) : info_type(info_type), type(type) {
47694
- }
47695
- MergeInfoType info_type;
47696
- LogicalType type;
47697
- };
47698
-
47699
- struct ScalarMergeInfo : public MergeInfo {
47700
- MergeOrder &order;
47701
- idx_t &pos;
47702
- SelectionVector result;
47703
-
47704
- ScalarMergeInfo(MergeOrder &order, LogicalType type, idx_t &pos)
47705
- : MergeInfo(MergeInfoType::SCALAR_MERGE_INFO, type), order(order), pos(pos), result(STANDARD_VECTOR_SIZE) {
47706
- }
47707
- };
47708
-
47709
- struct ChunkMergeInfo : public MergeInfo {
47710
- ChunkCollection &data_chunks;
47711
- vector<MergeOrder> &order_info;
47712
- bool found_match[STANDARD_VECTOR_SIZE];
47713
-
47714
- ChunkMergeInfo(ChunkCollection &data_chunks, vector<MergeOrder> &order_info)
47715
- : MergeInfo(MergeInfoType::CHUNK_MERGE_INFO, data_chunks.Types()[0]), data_chunks(data_chunks),
47716
- order_info(order_info) {
47717
- memset(found_match, 0, sizeof(found_match));
47718
- }
47719
- };
47720
-
47721
- struct MergeJoinComplex {
47722
- struct LessThan {
47723
- template <class T>
47724
- static idx_t Operation(ScalarMergeInfo &l, ScalarMergeInfo &r);
47725
- };
47726
- struct LessThanEquals {
47727
- template <class T>
47728
- static idx_t Operation(ScalarMergeInfo &l, ScalarMergeInfo &r);
47729
- };
47730
- struct GreaterThan {
47731
- template <class T>
47732
- static idx_t Operation(ScalarMergeInfo &l, ScalarMergeInfo &r) {
47733
- return LessThan::Operation<T>(r, l);
47734
- }
47735
- };
47736
- struct GreaterThanEquals {
47737
- template <class T>
47738
- static idx_t Operation(ScalarMergeInfo &l, ScalarMergeInfo &r) {
47739
- return LessThanEquals::Operation<T>(r, l);
47740
- }
47741
- };
47742
-
47743
- static idx_t Perform(MergeInfo &l, MergeInfo &r, ExpressionType comparison_type);
47744
- };
47745
-
47746
- struct MergeJoinSimple {
47747
- struct LessThan {
47748
- template <class T>
47749
- static idx_t Operation(ScalarMergeInfo &l, ChunkMergeInfo &r);
47750
- };
47751
- struct LessThanEquals {
47752
- template <class T>
47753
- static idx_t Operation(ScalarMergeInfo &l, ChunkMergeInfo &r);
47754
- };
47755
- struct GreaterThan {
47756
- template <class T>
47757
- static idx_t Operation(ScalarMergeInfo &l, ChunkMergeInfo &r);
47758
- };
47759
- struct GreaterThanEquals {
47760
- template <class T>
47761
- static idx_t Operation(ScalarMergeInfo &l, ChunkMergeInfo &r);
47762
- };
47763
-
47764
- static idx_t Perform(MergeInfo &l, MergeInfo &r, ExpressionType comparison);
47765
- };
47766
-
47767
- #define INSTANTIATE_MERGEJOIN_TEMPLATES(MJCLASS, OPNAME, L, R) \
47768
- template idx_t MJCLASS::OPNAME::Operation<int8_t>(L & l, R & r); \
47769
- template idx_t MJCLASS::OPNAME::Operation<int16_t>(L & l, R & r); \
47770
- template idx_t MJCLASS::OPNAME::Operation<int32_t>(L & l, R & r); \
47771
- template idx_t MJCLASS::OPNAME::Operation<int64_t>(L & l, R & r); \
47772
- template idx_t MJCLASS::OPNAME::Operation<uint8_t>(L & l, R & r); \
47773
- template idx_t MJCLASS::OPNAME::Operation<uint16_t>(L & l, R & r); \
47774
- template idx_t MJCLASS::OPNAME::Operation<uint32_t>(L & l, R & r); \
47775
- template idx_t MJCLASS::OPNAME::Operation<uint64_t>(L & l, R & r); \
47776
- template idx_t MJCLASS::OPNAME::Operation<hugeint_t>(L & l, R & r); \
47777
- template idx_t MJCLASS::OPNAME::Operation<float>(L & l, R & r); \
47778
- template idx_t MJCLASS::OPNAME::Operation<double>(L & l, R & r); \
47779
- template idx_t MJCLASS::OPNAME::Operation<interval_t>(L & l, R & r); \
47780
- template idx_t MJCLASS::OPNAME::Operation<string_t>(L & l, R & r);
47781
-
47782
- } // namespace duckdb
47783
-
47784
-
47785
-
47786
-
47787
- namespace duckdb {
47788
-
47789
- template <class MJ, class L_ARG, class R_ARG>
47790
- static idx_t MergeJoinSwitch(L_ARG &l, R_ARG &r) {
47791
- switch (l.type.InternalType()) {
47792
- case PhysicalType::BOOL:
47793
- case PhysicalType::INT8:
47794
- return MJ::template Operation<int8_t>(l, r);
47795
- case PhysicalType::INT16:
47796
- return MJ::template Operation<int16_t>(l, r);
47797
- case PhysicalType::INT32:
47798
- return MJ::template Operation<int32_t>(l, r);
47799
- case PhysicalType::INT64:
47800
- return MJ::template Operation<int64_t>(l, r);
47801
- case PhysicalType::UINT8:
47802
- return MJ::template Operation<uint8_t>(l, r);
47803
- case PhysicalType::UINT16:
47804
- return MJ::template Operation<uint16_t>(l, r);
47805
- case PhysicalType::UINT32:
47806
- return MJ::template Operation<uint32_t>(l, r);
47807
- case PhysicalType::UINT64:
47808
- return MJ::template Operation<uint64_t>(l, r);
47809
- case PhysicalType::INT128:
47810
- return MJ::template Operation<hugeint_t>(l, r);
47811
- case PhysicalType::FLOAT:
47812
- return MJ::template Operation<float>(l, r);
47813
- case PhysicalType::DOUBLE:
47814
- return MJ::template Operation<double>(l, r);
47815
- case PhysicalType::INTERVAL:
47816
- return MJ::template Operation<interval_t>(l, r);
47817
- case PhysicalType::VARCHAR:
47818
- return MJ::template Operation<string_t>(l, r);
47819
- default:
47820
- throw InternalException("Type not implemented for merge join!");
47821
- }
47822
- }
47823
-
47824
- template <class T, class L_ARG, class R_ARG>
47825
- static idx_t MergeJoinComparisonSwitch(L_ARG &l, R_ARG &r, ExpressionType comparison_type) {
47826
- switch (comparison_type) {
47827
- case ExpressionType::COMPARE_LESSTHAN:
47828
- return MergeJoinSwitch<typename T::LessThan, L_ARG, R_ARG>(l, r);
47829
- case ExpressionType::COMPARE_LESSTHANOREQUALTO:
47830
- return MergeJoinSwitch<typename T::LessThanEquals, L_ARG, R_ARG>(l, r);
47831
- case ExpressionType::COMPARE_GREATERTHAN:
47832
- return MergeJoinSwitch<typename T::GreaterThan, L_ARG, R_ARG>(l, r);
47833
- case ExpressionType::COMPARE_GREATERTHANOREQUALTO:
47834
- return MergeJoinSwitch<typename T::GreaterThanEquals, L_ARG, R_ARG>(l, r);
47835
- default:
47836
- throw InternalException("Unimplemented comparison type for merge join!");
47837
- }
47838
- }
47839
-
47840
- idx_t MergeJoinComplex::Perform(MergeInfo &l, MergeInfo &r, ExpressionType comparison_type) {
47841
- D_ASSERT(l.info_type == MergeInfoType::SCALAR_MERGE_INFO && r.info_type == MergeInfoType::SCALAR_MERGE_INFO);
47842
- auto &left = (ScalarMergeInfo &)l;
47843
- auto &right = (ScalarMergeInfo &)r;
47844
- D_ASSERT(left.type == right.type);
47845
- if (left.order.count == 0 || right.order.count == 0) {
47846
- return 0;
47847
- }
47848
- return MergeJoinComparisonSwitch<MergeJoinComplex, ScalarMergeInfo, ScalarMergeInfo>(left, right, comparison_type);
47849
- }
47850
-
47851
- idx_t MergeJoinSimple::Perform(MergeInfo &l, MergeInfo &r, ExpressionType comparison_type) {
47852
- D_ASSERT(l.info_type == MergeInfoType::SCALAR_MERGE_INFO && r.info_type == MergeInfoType::CHUNK_MERGE_INFO);
47853
- auto &left = (ScalarMergeInfo &)l;
47854
- auto &right = (ChunkMergeInfo &)r;
47855
- D_ASSERT(left.type == right.type);
47856
- if (left.order.count == 0 || right.data_chunks.Count() == 0) {
47857
- return 0;
47858
- }
47859
- return MergeJoinComparisonSwitch<MergeJoinSimple, ScalarMergeInfo, ChunkMergeInfo>(left, right, comparison_type);
47860
- }
47861
-
47862
- } // namespace duckdb
47863
-
47864
-
47865
-
47866
-
47867
-
47868
- namespace duckdb {
47869
-
47870
- template <class T, class OP>
47871
- idx_t MergeJoinComplexLessThan(ScalarMergeInfo &l, ScalarMergeInfo &r) {
47872
- if (r.pos >= r.order.count) {
47873
- return 0;
47874
- }
47875
- auto ldata = (T *)l.order.vdata.data;
47876
- auto rdata = (T *)r.order.vdata.data;
47877
- auto &lorder = l.order.order;
47878
- auto &rorder = r.order.order;
47879
- idx_t result_count = 0;
47880
- while (true) {
47881
- if (l.pos < l.order.count) {
47882
- auto lidx = lorder.get_index(l.pos);
47883
- auto ridx = rorder.get_index(r.pos);
47884
- auto dlidx = l.order.vdata.sel->get_index(lidx);
47885
- auto dridx = r.order.vdata.sel->get_index(ridx);
47886
- if (OP::Operation(ldata[dlidx], rdata[dridx])) {
47887
- // left side smaller: found match
47888
- l.result.set_index(result_count, lidx);
47889
- r.result.set_index(result_count, ridx);
47890
- result_count++;
47891
- // move left side forward
47892
- l.pos++;
47893
- if (result_count == STANDARD_VECTOR_SIZE) {
47894
- // out of space!
47895
- break;
47896
- }
47897
- continue;
47898
- }
47899
- }
47900
- // right side smaller or equal, or left side exhausted: move
47901
- // right pointer forward reset left side to start
47902
- l.pos = 0;
47903
- r.pos++;
47904
- if (r.pos == r.order.count) {
47905
- break;
47906
- }
47907
- }
47908
- return result_count;
47909
- }
47910
-
47911
- template <class T>
47912
- idx_t MergeJoinComplex::LessThan::Operation(ScalarMergeInfo &l, ScalarMergeInfo &r) {
47913
- return MergeJoinComplexLessThan<T, duckdb::LessThan>(l, r);
47914
- }
47915
-
47916
- template <class T>
47917
- idx_t MergeJoinComplex::LessThanEquals::Operation(ScalarMergeInfo &l, ScalarMergeInfo &r) {
47918
- return MergeJoinComplexLessThan<T, duckdb::LessThanEquals>(l, r);
47919
- }
47920
-
47921
- INSTANTIATE_MERGEJOIN_TEMPLATES(MergeJoinComplex, LessThan, ScalarMergeInfo, ScalarMergeInfo)
47922
- INSTANTIATE_MERGEJOIN_TEMPLATES(MergeJoinComplex, LessThanEquals, ScalarMergeInfo, ScalarMergeInfo)
47923
-
47924
- } // namespace duckdb
47925
-
47926
-
47927
-
47928
-
47929
-
47930
- namespace duckdb {
47931
-
47932
- template <class T, class OP>
47933
- static idx_t MergeJoinSimpleGreaterThan(ScalarMergeInfo &l, ChunkMergeInfo &r) {
47934
- auto ldata = (T *)l.order.vdata.data;
47935
- auto &lorder = l.order.order;
47936
- l.pos = l.order.count;
47937
- for (idx_t chunk_idx = 0; chunk_idx < r.order_info.size(); chunk_idx++) {
47938
- // we only care about the SMALLEST value in each of the RHS
47939
- // because we want to figure out if they are greater than [or equal] to ANY value
47940
- // get the smallest value from the RHS
47941
- auto &rorder = r.order_info[chunk_idx];
47942
- auto rdata = (T *)rorder.vdata.data;
47943
- auto min_r_value = rdata[rorder.vdata.sel->get_index(rorder.order.get_index(0))];
47944
- // now we start from the current lpos value and check if we found a new value that is [>= OR >] the min RHS
47945
- // value
47946
- while (true) {
47947
- auto lidx = lorder.get_index(l.pos - 1);
47948
- auto dlidx = l.order.vdata.sel->get_index(lidx);
47949
- if (OP::Operation(ldata[dlidx], min_r_value)) {
47950
- // found a match for lpos, set it in the found_match vector
47951
- r.found_match[lidx] = true;
47952
- l.pos--;
47953
- if (l.pos == 0) {
47954
- // early out: we exhausted the entire LHS and they all match
47955
- return 0;
47956
- }
47957
- } else {
47958
- // we found no match: any subsequent value from the LHS we scan now will be smaller and thus also not
47959
- // match move to the next RHS chunk
47960
- break;
47961
- }
47962
- }
47963
- }
47964
- return 0;
47965
- }
47966
- template <class T>
47967
- idx_t MergeJoinSimple::GreaterThan::Operation(ScalarMergeInfo &l, ChunkMergeInfo &r) {
47968
- return MergeJoinSimpleGreaterThan<T, duckdb::GreaterThan>(l, r);
47969
- }
47970
-
47971
- template <class T>
47972
- idx_t MergeJoinSimple::GreaterThanEquals::Operation(ScalarMergeInfo &l, ChunkMergeInfo &r) {
47973
- return MergeJoinSimpleGreaterThan<T, duckdb::GreaterThanEquals>(l, r);
47974
- }
47975
-
47976
- template <class T, class OP>
47977
- static idx_t MergeJoinSimpleLessThan(ScalarMergeInfo &l, ChunkMergeInfo &r) {
47978
- auto ldata = (T *)l.order.vdata.data;
47979
- auto &lorder = l.order.order;
47980
- l.pos = 0;
47981
- for (idx_t chunk_idx = 0; chunk_idx < r.order_info.size(); chunk_idx++) {
47982
- // we only care about the BIGGEST value in each of the RHS
47983
- // because we want to figure out if they are less than [or equal] to ANY value
47984
- // get the biggest value from the RHS
47985
- auto &rorder = r.order_info[chunk_idx];
47986
- auto rdata = (T *)rorder.vdata.data;
47987
- auto max_r_value = rdata[rorder.vdata.sel->get_index(rorder.order.get_index(rorder.count - 1))];
47988
- // now we start from the current lpos value and check if we found a new value that is [<= OR <] the max RHS
47989
- // value
47990
- while (true) {
47991
- auto lidx = lorder.get_index(l.pos);
47992
- auto dlidx = l.order.vdata.sel->get_index(lidx);
47993
- if (OP::Operation(ldata[dlidx], max_r_value)) {
47994
- // found a match for lpos, set it in the found_match vector
47995
- r.found_match[lidx] = true;
47996
- l.pos++;
47997
- if (l.pos == l.order.count) {
47998
- // early out: we exhausted the entire LHS and they all match
47999
- return 0;
48000
- }
48001
- } else {
48002
- // we found no match: any subsequent value from the LHS we scan now will be bigger and thus also not
48003
- // match move to the next RHS chunk
48004
- break;
48005
- }
48006
- }
48007
- }
48008
- return 0;
48009
- }
48010
-
48011
- template <class T>
48012
- idx_t MergeJoinSimple::LessThan::Operation(ScalarMergeInfo &l, ChunkMergeInfo &r) {
48013
- return MergeJoinSimpleLessThan<T, duckdb::LessThan>(l, r);
48014
- }
48015
-
48016
- template <class T>
48017
- idx_t MergeJoinSimple::LessThanEquals::Operation(ScalarMergeInfo &l, ChunkMergeInfo &r) {
48018
- return MergeJoinSimpleLessThan<T, duckdb::LessThanEquals>(l, r);
48019
- }
48020
-
48021
- INSTANTIATE_MERGEJOIN_TEMPLATES(MergeJoinSimple, LessThan, ScalarMergeInfo, ChunkMergeInfo)
48022
- INSTANTIATE_MERGEJOIN_TEMPLATES(MergeJoinSimple, LessThanEquals, ScalarMergeInfo, ChunkMergeInfo)
48023
- INSTANTIATE_MERGEJOIN_TEMPLATES(MergeJoinSimple, GreaterThan, ScalarMergeInfo, ChunkMergeInfo)
48024
- INSTANTIATE_MERGEJOIN_TEMPLATES(MergeJoinSimple, GreaterThanEquals, ScalarMergeInfo, ChunkMergeInfo)
48025
-
48026
47668
  } // namespace duckdb
48027
47669
 
48028
47670
  //===----------------------------------------------------------------------===//
@@ -55010,6 +54652,8 @@ void PhysicalNestedLoopJoin::GetData(ExecutionContext &context, DataChunk &chunk
55010
54652
 
55011
54653
  namespace duckdb {
55012
54654
 
54655
+ class MergeJoinGlobalState;
54656
+
55013
54657
  //! PhysicalPiecewiseMergeJoin represents a piecewise merge loop join between
55014
54658
  //! two tables
55015
54659
  class PhysicalPiecewiseMergeJoin : public PhysicalComparisonJoin {
@@ -55019,6 +54663,8 @@ public:
55019
54663
  idx_t estimated_cardinality);
55020
54664
 
55021
54665
  vector<LogicalType> join_key_types;
54666
+ vector<BoundOrderByNode> lhs_orders;
54667
+ vector<BoundOrderByNode> rhs_orders;
55022
54668
 
55023
54669
  public:
55024
54670
  // Operator Interface
@@ -55057,6 +54703,9 @@ public:
55057
54703
  SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
55058
54704
  GlobalSinkState &gstate) const override;
55059
54705
 
54706
+ //! Schedules tasks to merge sort the RHS data during the Finalize phase
54707
+ static void ScheduleMergeTasks(Pipeline &pipeline, Event &event, MergeJoinGlobalState &state);
54708
+
55060
54709
  bool IsSink() const override {
55061
54710
  return true;
55062
54711
  }
@@ -55081,6 +54730,11 @@ private:
55081
54730
 
55082
54731
 
55083
54732
 
54733
+
54734
+
54735
+
54736
+
54737
+
55084
54738
  namespace duckdb {
55085
54739
 
55086
54740
  PhysicalPiecewiseMergeJoin::PhysicalPiecewiseMergeJoin(LogicalOperator &op, unique_ptr<PhysicalOperator> left,
@@ -55091,10 +54745,27 @@ PhysicalPiecewiseMergeJoin::PhysicalPiecewiseMergeJoin(LogicalOperator &op, uniq
55091
54745
  // for now we only support one condition!
55092
54746
  D_ASSERT(conditions.size() == 1);
55093
54747
  for (auto &cond : conditions) {
55094
- // COMPARE NOT EQUAL not supported with merge join
55095
- D_ASSERT(cond.comparison != ExpressionType::COMPARE_NOTEQUAL);
55096
54748
  D_ASSERT(cond.left->return_type == cond.right->return_type);
55097
54749
  join_key_types.push_back(cond.left->return_type);
54750
+
54751
+ // Convert the conditions to sort orders
54752
+ auto left = cond.left->Copy();
54753
+ auto right = cond.right->Copy();
54754
+ switch (cond.comparison) {
54755
+ case ExpressionType::COMPARE_LESSTHAN:
54756
+ case ExpressionType::COMPARE_LESSTHANOREQUALTO:
54757
+ lhs_orders.emplace_back(BoundOrderByNode(OrderType::ASCENDING, OrderByNullType::NULLS_LAST, move(left)));
54758
+ rhs_orders.emplace_back(BoundOrderByNode(OrderType::ASCENDING, OrderByNullType::NULLS_LAST, move(right)));
54759
+ break;
54760
+ case ExpressionType::COMPARE_GREATERTHAN:
54761
+ case ExpressionType::COMPARE_GREATERTHANOREQUALTO:
54762
+ lhs_orders.emplace_back(BoundOrderByNode(OrderType::DESCENDING, OrderByNullType::NULLS_LAST, move(left)));
54763
+ rhs_orders.emplace_back(BoundOrderByNode(OrderType::DESCENDING, OrderByNullType::NULLS_LAST, move(right)));
54764
+ break;
54765
+ default:
54766
+ // COMPARE NOT EQUAL not supported with merge join
54767
+ throw NotImplementedException("Unimplemented join type for merge join");
54768
+ }
55098
54769
  }
55099
54770
  children.push_back(move(left));
55100
54771
  children.push_back(move(right));
@@ -55103,114 +54774,242 @@ PhysicalPiecewiseMergeJoin::PhysicalPiecewiseMergeJoin(LogicalOperator &op, uniq
55103
54774
  //===--------------------------------------------------------------------===//
55104
54775
  // Sink
55105
54776
  //===--------------------------------------------------------------------===//
55106
- class MergeJoinLocalState : public LocalSinkState {
54777
+ class MergeJoinGlobalState : public GlobalSinkState {
55107
54778
  public:
55108
- explicit MergeJoinLocalState(const vector<JoinCondition> &conditions) {
55109
- vector<LogicalType> condition_types;
55110
- for (auto &cond : conditions) {
55111
- rhs_executor.AddExpression(*cond.right);
55112
- condition_types.push_back(cond.right->return_type);
55113
- }
55114
- join_keys.Initialize(condition_types);
54779
+ MergeJoinGlobalState(BufferManager &buffer_manager, const vector<BoundOrderByNode> &orders, RowLayout &rhs_layout)
54780
+ : rhs_global_sort_state(buffer_manager, orders, rhs_layout), rhs_has_null(0), rhs_count(0),
54781
+ memory_per_thread(0) {
55115
54782
  }
55116
54783
 
55117
- //! The chunk holding the right condition
55118
- DataChunk join_keys;
55119
- //! The executor of the RHS condition
55120
- ExpressionExecutor rhs_executor;
55121
- };
55122
-
55123
- class MergeJoinGlobalState : public GlobalSinkState {
55124
- public:
55125
- MergeJoinGlobalState() : has_null(false), right_outer_position(0) {
54784
+ inline idx_t Count() const {
54785
+ return rhs_count;
55126
54786
  }
55127
54787
 
55128
- mutex mj_lock;
55129
- //! The materialized data of the RHS
55130
- ChunkCollection right_chunks;
55131
- //! The materialized join keys of the RHS
55132
- ChunkCollection right_conditions;
55133
- //! The join orders of the RHS
55134
- vector<MergeOrder> right_orders;
55135
- //! Whether or not the RHS of the nested loop join has NULL values
55136
- bool has_null;
54788
+ //! The lock for updating the global state
54789
+ mutex lock;
54790
+ //! Global sort state
54791
+ GlobalSortState rhs_global_sort_state;
54792
+ //! Whether or not the RHS has NULL values
54793
+ idx_t rhs_has_null;
54794
+ //! The total number of rows in the RHS
54795
+ idx_t rhs_count;
55137
54796
  //! A bool indicating for each tuple in the RHS if they found a match (only used in FULL OUTER JOIN)
55138
- unique_ptr<bool[]> right_found_match;
55139
- //! The position in the RHS in the final scan of the FULL OUTER JOIN
55140
- idx_t right_outer_position;
54797
+ unique_ptr<bool[]> rhs_found_match;
54798
+ //! Memory usage per thread
54799
+ idx_t memory_per_thread;
55141
54800
  };
55142
54801
 
55143
54802
  unique_ptr<GlobalSinkState> PhysicalPiecewiseMergeJoin::GetGlobalSinkState(ClientContext &context) const {
55144
- return make_unique<MergeJoinGlobalState>();
54803
+ // Get the payload layout from the rhs types
54804
+ RowLayout rhs_layout;
54805
+ auto types = children[1]->types;
54806
+ rhs_layout.Initialize(types);
54807
+ auto state = make_unique<MergeJoinGlobalState>(BufferManager::GetBufferManager(context), rhs_orders, rhs_layout);
54808
+ // Set external (can be force with the PRAGMA)
54809
+ auto &config = ClientConfig::GetConfig(context);
54810
+ state->rhs_global_sort_state.external = config.force_external;
54811
+ // Memory usage per thread should scale with max mem / num threads
54812
+ // We take 1/4th of this, to be conservative
54813
+ idx_t max_memory = BufferManager::GetBufferManager(context).GetMaxMemory();
54814
+ idx_t num_threads = TaskScheduler::GetScheduler(context).NumberOfThreads();
54815
+ state->memory_per_thread = (max_memory / num_threads) / 4;
54816
+ return move(state);
55145
54817
  }
55146
54818
 
54819
+ static idx_t CountValid(Vector &v, const idx_t count) {
54820
+ idx_t valid = 0;
54821
+
54822
+ VectorData vdata;
54823
+ v.Orrify(count, vdata);
54824
+ if (vdata.validity.AllValid()) {
54825
+ return count;
54826
+ }
54827
+ switch (v.GetVectorType()) {
54828
+ case VectorType::FLAT_VECTOR:
54829
+ valid += vdata.validity.CountValid(count);
54830
+ break;
54831
+ case VectorType::CONSTANT_VECTOR:
54832
+ valid += vdata.validity.CountValid(1) * count;
54833
+ break;
54834
+ default:
54835
+ for (idx_t i = 0; i < count; ++i) {
54836
+ const auto row_idx = vdata.sel->get_index(i);
54837
+ valid += int(vdata.validity.RowIsValid(row_idx));
54838
+ }
54839
+ break;
54840
+ }
54841
+
54842
+ return valid;
54843
+ }
54844
+
54845
+ class MergeJoinLocalState : public LocalSinkState {
54846
+ public:
54847
+ explicit MergeJoinLocalState() : rhs_has_null(0), rhs_count(0) {
54848
+ }
54849
+
54850
+ //! The local sort state
54851
+ LocalSortState rhs_local_sort_state;
54852
+ //! Local copy of the sorting expression executor
54853
+ ExpressionExecutor rhs_executor;
54854
+ //! Holds a vector of incoming sorting columns
54855
+ DataChunk rhs_keys;
54856
+ //! Whether or not the RHS has NULL values
54857
+ idx_t rhs_has_null;
54858
+ //! The total number of rows in the RHS
54859
+ idx_t rhs_count;
54860
+ };
54861
+
55147
54862
  unique_ptr<LocalSinkState> PhysicalPiecewiseMergeJoin::GetLocalSinkState(ExecutionContext &context) const {
55148
- return make_unique<MergeJoinLocalState>(conditions);
54863
+ auto result = make_unique<MergeJoinLocalState>();
54864
+ // Initialize order clause expression executor and DataChunk
54865
+ vector<LogicalType> types;
54866
+ for (auto &order : rhs_orders) {
54867
+ types.push_back(order.expression->return_type);
54868
+ result->rhs_executor.AddExpression(*order.expression);
54869
+ }
54870
+ result->rhs_keys.Initialize(types);
54871
+ return move(result);
55149
54872
  }
55150
54873
 
55151
- SinkResultType PhysicalPiecewiseMergeJoin::Sink(ExecutionContext &context, GlobalSinkState &state,
55152
- LocalSinkState &lstate, DataChunk &input) const {
55153
- auto &gstate = (MergeJoinGlobalState &)state;
55154
- auto &mj_state = (MergeJoinLocalState &)lstate;
54874
+ SinkResultType PhysicalPiecewiseMergeJoin::Sink(ExecutionContext &context, GlobalSinkState &gstate_p,
54875
+ LocalSinkState &lstate_p, DataChunk &input) const {
54876
+ auto &gstate = (MergeJoinGlobalState &)gstate_p;
54877
+ auto &lstate = (MergeJoinLocalState &)lstate_p;
55155
54878
 
55156
- // resolve the join keys for this chunk
55157
- mj_state.rhs_executor.SetChunk(input);
54879
+ auto &global_sort_state = gstate.rhs_global_sort_state;
54880
+ auto &local_sort_state = lstate.rhs_local_sort_state;
55158
54881
 
55159
- mj_state.join_keys.Reset();
55160
- mj_state.join_keys.SetCardinality(input);
55161
- for (idx_t k = 0; k < conditions.size(); k++) {
55162
- // resolve the join key
55163
- mj_state.rhs_executor.ExecuteExpression(k, mj_state.join_keys.data[k]);
54882
+ // Initialize local state (if necessary)
54883
+ if (!local_sort_state.initialized) {
54884
+ local_sort_state.Initialize(global_sort_state, BufferManager::GetBufferManager(context.client));
54885
+ }
54886
+
54887
+ // Obtain sorting columns
54888
+ auto &join_keys = lstate.rhs_keys;
54889
+ join_keys.Reset();
54890
+ lstate.rhs_executor.Execute(input, join_keys);
54891
+
54892
+ // Count the NULLs so we can exclude them later
54893
+ // TODO: Sort any comparison NULLs to the end using an initial sort column
54894
+ const auto count = join_keys.size();
54895
+ for (auto &key : join_keys.data) {
54896
+ lstate.rhs_has_null += (count - CountValid(key, count));
54897
+ }
54898
+ lstate.rhs_count += count;
54899
+
54900
+ // Sink the data into the local sort state
54901
+ local_sort_state.SinkChunk(join_keys, input);
54902
+
54903
+ // When sorting data reaches a certain size, we sort it
54904
+ if (local_sort_state.SizeInBytes() >= gstate.memory_per_thread) {
54905
+ local_sort_state.Sort(global_sort_state, true);
55164
54906
  }
55165
- // append the join keys and the chunk to the chunk collection
55166
- lock_guard<mutex> mj_guard(gstate.mj_lock);
55167
- gstate.right_chunks.Append(input);
55168
- gstate.right_conditions.Append(mj_state.join_keys);
55169
54907
  return SinkResultType::NEED_MORE_INPUT;
55170
54908
  }
55171
54909
 
55172
- void PhysicalPiecewiseMergeJoin::Combine(ExecutionContext &context, GlobalSinkState &gstate,
55173
- LocalSinkState &lstate) const {
55174
- auto &state = (MergeJoinLocalState &)lstate;
54910
+ void PhysicalPiecewiseMergeJoin::Combine(ExecutionContext &context, GlobalSinkState &gstate_p,
54911
+ LocalSinkState &lstate_p) const {
54912
+ auto &gstate = (MergeJoinGlobalState &)gstate_p;
54913
+ auto &lstate = (MergeJoinLocalState &)lstate_p;
54914
+ gstate.rhs_global_sort_state.AddLocalState(lstate.rhs_local_sort_state);
54915
+ lock_guard<mutex> locked(gstate.lock);
54916
+ gstate.rhs_has_null += lstate.rhs_has_null;
54917
+ gstate.rhs_count += lstate.rhs_count;
55175
54918
  auto &client_profiler = QueryProfiler::Get(context.client);
55176
54919
 
55177
- context.thread.profiler.Flush(this, &state.rhs_executor, "rhs_executor", 1);
54920
+ context.thread.profiler.Flush(this, &lstate.rhs_executor, "rhs_executor", 1);
55178
54921
  client_profiler.Flush(context.thread.profiler);
55179
54922
  }
55180
54923
 
55181
54924
  //===--------------------------------------------------------------------===//
55182
54925
  // Finalize
55183
54926
  //===--------------------------------------------------------------------===//
55184
- static void OrderVector(Vector &vector, idx_t count, MergeOrder &order);
54927
+ class MergeJoinFinalizeTask : public ExecutorTask {
54928
+ public:
54929
+ MergeJoinFinalizeTask(shared_ptr<Event> event_p, ClientContext &context, MergeJoinGlobalState &state)
54930
+ : ExecutorTask(context), event(move(event_p)), context(context), state(state) {
54931
+ }
54932
+
54933
+ TaskExecutionResult ExecuteTask(TaskExecutionMode mode) override {
54934
+ // Initialize merge sorted and iterate until done
54935
+ auto &global_sort_state = state.rhs_global_sort_state;
54936
+ MergeSorter merge_sorter(global_sort_state, BufferManager::GetBufferManager(context));
54937
+ merge_sorter.PerformInMergeRound();
54938
+ event->FinishTask();
54939
+
54940
+ return TaskExecutionResult::TASK_FINISHED;
54941
+ }
54942
+
54943
+ private:
54944
+ shared_ptr<Event> event;
54945
+ ClientContext &context;
54946
+ MergeJoinGlobalState &state;
54947
+ };
54948
+
54949
+ class MergeJoinFinalizeEvent : public Event {
54950
+ public:
54951
+ MergeJoinFinalizeEvent(MergeJoinGlobalState &gstate_p, Pipeline &pipeline_p)
54952
+ : Event(pipeline_p.executor), gstate(gstate_p), pipeline(pipeline_p) {
54953
+ }
54954
+
54955
+ MergeJoinGlobalState &gstate;
54956
+ Pipeline &pipeline;
54957
+
54958
+ public:
54959
+ void Schedule() override {
54960
+ auto &context = pipeline.GetClientContext();
54961
+
54962
+ // Schedule tasks equal to the number of threads, which will each merge multiple partitions
54963
+ auto &ts = TaskScheduler::GetScheduler(context);
54964
+ idx_t num_threads = ts.NumberOfThreads();
54965
+
54966
+ vector<unique_ptr<Task>> merge_tasks;
54967
+ for (idx_t tnum = 0; tnum < num_threads; tnum++) {
54968
+ merge_tasks.push_back(make_unique<MergeJoinFinalizeTask>(shared_from_this(), context, gstate));
54969
+ }
54970
+ SetTasks(move(merge_tasks));
54971
+ }
54972
+
54973
+ void FinishEvent() override {
54974
+ auto &global_sort_state = gstate.rhs_global_sort_state;
54975
+
54976
+ global_sort_state.CompleteMergeRound(true);
54977
+ if (global_sort_state.sorted_blocks.size() > 1) {
54978
+ // Multiple blocks remaining: Schedule the next round
54979
+ PhysicalPiecewiseMergeJoin::ScheduleMergeTasks(pipeline, *this, gstate);
54980
+ }
54981
+ }
54982
+ };
54983
+
54984
+ void PhysicalPiecewiseMergeJoin::ScheduleMergeTasks(Pipeline &pipeline, Event &event, MergeJoinGlobalState &gstate) {
54985
+ // Initialize global sort state for a round of merging
54986
+ gstate.rhs_global_sort_state.InitializeMergeRound();
54987
+ auto new_event = make_shared<MergeJoinFinalizeEvent>(gstate, pipeline);
54988
+ event.InsertEvent(move(new_event));
54989
+ }
55185
54990
 
55186
54991
  SinkFinalizeType PhysicalPiecewiseMergeJoin::Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
55187
54992
  GlobalSinkState &gstate_p) const {
55188
54993
  auto &gstate = (MergeJoinGlobalState &)gstate_p;
55189
- if (gstate.right_conditions.ChunkCount() > 0) {
55190
- // now order all the chunks
55191
- gstate.right_orders.resize(gstate.right_conditions.ChunkCount());
55192
- for (idx_t i = 0; i < gstate.right_conditions.ChunkCount(); i++) {
55193
- auto &chunk_to_order = gstate.right_conditions.GetChunk(i);
55194
- D_ASSERT(chunk_to_order.ColumnCount() == 1);
55195
- for (idx_t col_idx = 0; col_idx < chunk_to_order.ColumnCount(); col_idx++) {
55196
- OrderVector(chunk_to_order.data[col_idx], chunk_to_order.size(), gstate.right_orders[i]);
55197
- if (gstate.right_orders[i].count < chunk_to_order.size()) {
55198
- // the amount of entries in the order vector is smaller than the amount of entries in the vector
55199
- // this only happens if there are NULL values in the right-hand side
55200
- // hence we set the has_null to true (this is required for the MARK join)
55201
- gstate.has_null = true;
55202
- }
55203
- }
55204
- }
55205
- }
54994
+ auto &global_sort_state = gstate.rhs_global_sort_state;
54995
+
55206
54996
  if (IsRightOuterJoin(join_type)) {
55207
54997
  // for FULL/RIGHT OUTER JOIN, initialize found_match to false for every tuple
55208
- gstate.right_found_match = unique_ptr<bool[]>(new bool[gstate.right_chunks.Count()]);
55209
- memset(gstate.right_found_match.get(), 0, sizeof(bool) * gstate.right_chunks.Count());
54998
+ gstate.rhs_found_match = unique_ptr<bool[]>(new bool[gstate.Count()]);
54999
+ memset(gstate.rhs_found_match.get(), 0, sizeof(bool) * gstate.Count());
55210
55000
  }
55211
- if (gstate.right_chunks.Count() == 0 && EmptyResultIfRHSIsEmpty()) {
55001
+ if (global_sort_state.sorted_blocks.empty() && EmptyResultIfRHSIsEmpty()) {
55002
+ // Empty input!
55212
55003
  return SinkFinalizeType::NO_OUTPUT_POSSIBLE;
55213
55004
  }
55005
+
55006
+ // Prepare for merge sort phase
55007
+ global_sort_state.PrepareMergePhase();
55008
+
55009
+ // Start the merge phase or finish if a merge is not necessary
55010
+ if (global_sort_state.sorted_blocks.size() > 1) {
55011
+ PhysicalPiecewiseMergeJoin::ScheduleMergeTasks(pipeline, event, gstate);
55012
+ }
55214
55013
  return SinkFinalizeType::READY;
55215
55014
  }
55216
55015
 
@@ -55219,43 +55018,75 @@ SinkFinalizeType PhysicalPiecewiseMergeJoin::Finalize(Pipeline &pipeline, Event
55219
55018
  //===--------------------------------------------------------------------===//
55220
55019
  class PiecewiseMergeJoinState : public OperatorState {
55221
55020
  public:
55222
- explicit PiecewiseMergeJoinState(const PhysicalPiecewiseMergeJoin &op)
55223
- : op(op), first_fetch(true), finished(true), left_position(0), right_position(0), right_chunk_index(0) {
55021
+ explicit PiecewiseMergeJoinState(const PhysicalPiecewiseMergeJoin &op, BufferManager &buffer_manager,
55022
+ bool force_external)
55023
+ : op(op), buffer_manager(buffer_manager), force_external(force_external), left_position(0), first_fetch(true),
55024
+ finished(true), right_position(0), right_chunk_index(0) {
55224
55025
  vector<LogicalType> condition_types;
55225
55026
  for (auto &cond : op.conditions) {
55226
55027
  lhs_executor.AddExpression(*cond.left);
55227
55028
  condition_types.push_back(cond.left->return_type);
55228
55029
  }
55229
- join_keys.Initialize(condition_types);
55030
+ lhs_keys.Initialize(condition_types);
55230
55031
  if (IsLeftOuterJoin(op.join_type)) {
55231
- left_found_match = unique_ptr<bool[]>(new bool[STANDARD_VECTOR_SIZE]);
55232
- memset(left_found_match.get(), 0, sizeof(bool) * STANDARD_VECTOR_SIZE);
55032
+ lhs_found_match = unique_ptr<bool[]>(new bool[STANDARD_VECTOR_SIZE]);
55033
+ memset(lhs_found_match.get(), 0, sizeof(bool) * STANDARD_VECTOR_SIZE);
55233
55034
  }
55035
+ lhs_layout.Initialize(op.children[0]->types);
55234
55036
  }
55235
55037
 
55236
55038
  const PhysicalPiecewiseMergeJoin &op;
55039
+ BufferManager &buffer_manager;
55040
+ bool force_external;
55041
+
55042
+ // Block sorting
55043
+ DataChunk lhs_keys;
55044
+ ExpressionExecutor lhs_executor;
55045
+ unique_ptr<bool[]> lhs_found_match;
55046
+ RowLayout lhs_layout;
55047
+ unique_ptr<LocalSortState> lhs_local_state;
55048
+ unique_ptr<GlobalSortState> lhs_global_state;
55049
+ idx_t lhs_count;
55050
+ idx_t lhs_has_null;
55051
+
55052
+ // Simple scans
55053
+ idx_t left_position;
55054
+
55055
+ // Complex scans
55237
55056
  bool first_fetch;
55238
55057
  bool finished;
55239
- idx_t left_position;
55240
55058
  idx_t right_position;
55241
55059
  idx_t right_chunk_index;
55242
- DataChunk left_chunk;
55243
- DataChunk join_keys;
55244
- MergeOrder left_orders;
55245
- //! The executor of the RHS condition
55246
- ExpressionExecutor lhs_executor;
55247
- unique_ptr<bool[]> left_found_match;
55060
+ idx_t right_base;
55248
55061
 
55249
55062
  public:
55250
55063
  void ResolveJoinKeys(DataChunk &input) {
55251
55064
  // resolve the join keys for the input
55252
- join_keys.Reset();
55253
- lhs_executor.SetChunk(input);
55254
- join_keys.SetCardinality(input);
55255
- for (idx_t k = 0; k < op.conditions.size(); k++) {
55256
- lhs_executor.ExecuteExpression(k, join_keys.data[k]);
55257
- // sort by join key
55258
- OrderVector(join_keys.data[k], join_keys.size(), left_orders);
55065
+ lhs_keys.Reset();
55066
+ lhs_executor.Execute(input, lhs_keys);
55067
+
55068
+ // Count the NULLs so we can exclude them later
55069
+ // TODO: Sort any multi-comparison NULLs to the end using an initial sort column
55070
+ lhs_count = lhs_keys.size();
55071
+ for (auto &key : lhs_keys.data) {
55072
+ lhs_has_null = lhs_count - CountValid(key, lhs_count);
55073
+ break;
55074
+ }
55075
+
55076
+ // sort by join key
55077
+ lhs_global_state = make_unique<GlobalSortState>(buffer_manager, op.lhs_orders, lhs_layout);
55078
+ lhs_local_state = make_unique<LocalSortState>();
55079
+ lhs_local_state->Initialize(*lhs_global_state, buffer_manager);
55080
+ lhs_local_state->SinkChunk(lhs_keys, input);
55081
+
55082
+ // Set external (can be force with the PRAGMA)
55083
+ lhs_global_state->external = force_external;
55084
+ lhs_global_state->AddLocalState(*lhs_local_state);
55085
+ lhs_global_state->PrepareMergePhase();
55086
+ while (lhs_global_state->sorted_blocks.size() > 1) {
55087
+ MergeSorter merge_sorter(*lhs_global_state, buffer_manager);
55088
+ merge_sorter.PerformInMergeRound();
55089
+ lhs_global_state->CompleteMergeRound();
55259
55090
  }
55260
55091
  }
55261
55092
 
@@ -55265,7 +55096,176 @@ public:
55265
55096
  };
55266
55097
 
55267
55098
  unique_ptr<OperatorState> PhysicalPiecewiseMergeJoin::GetOperatorState(ClientContext &context) const {
55268
- return make_unique<PiecewiseMergeJoinState>(*this);
55099
+ auto &buffer_manager = BufferManager::GetBufferManager(context);
55100
+ auto &config = ClientConfig::GetConfig(context);
55101
+ return make_unique<PiecewiseMergeJoinState>(*this, buffer_manager, config.force_external);
55102
+ }
55103
+
55104
+ static inline idx_t SortedBlockNotNull(const idx_t base, const idx_t count, const idx_t not_null) {
55105
+ return MinValue(base + count, MaxValue(base, not_null)) - base;
55106
+ }
55107
+
55108
+ static int MergeJoinComparisonValue(ExpressionType comparison) {
55109
+ switch (comparison) {
55110
+ case ExpressionType::COMPARE_LESSTHAN:
55111
+ case ExpressionType::COMPARE_GREATERTHAN:
55112
+ return -1;
55113
+ case ExpressionType::COMPARE_LESSTHANOREQUALTO:
55114
+ case ExpressionType::COMPARE_GREATERTHANOREQUALTO:
55115
+ return 0;
55116
+ default:
55117
+ throw InternalException("Unimplemented comparison type for merge join!");
55118
+ }
55119
+ }
55120
+
55121
+ struct BlockMergeInfo {
55122
+ GlobalSortState &state;
55123
+ //! The block being scanned
55124
+ const idx_t block_idx;
55125
+ //! The start position being read from the block
55126
+ const idx_t base_idx;
55127
+ //! The number of not-NULL values in the block (they are at the end)
55128
+ const idx_t not_null;
55129
+ //! The current offset in the block
55130
+ idx_t &entry_idx;
55131
+ SelectionVector result;
55132
+
55133
+ BlockMergeInfo(GlobalSortState &state, idx_t block_idx, idx_t base_idx, idx_t &entry_idx, idx_t not_null)
55134
+ : state(state), block_idx(block_idx), base_idx(base_idx), not_null(not_null), entry_idx(entry_idx),
55135
+ result(STANDARD_VECTOR_SIZE) {
55136
+ }
55137
+ };
55138
+
55139
+ static void SliceSortedPayload(DataChunk &payload, BlockMergeInfo &info, const idx_t result_count, const idx_t vcount,
55140
+ const idx_t left_cols = 0) {
55141
+ // There should only be one sorted block if they have been sorted
55142
+ D_ASSERT(info.state.sorted_blocks.size() == 1);
55143
+ SBScanState read_state(info.state.buffer_manager, info.state);
55144
+ read_state.sb = info.state.sorted_blocks[0].get();
55145
+ auto &sorted_data = *read_state.sb->payload_data;
55146
+
55147
+ // We have to create pointers for the entire block
55148
+ // because unswizzle works on ranges not selections.
55149
+ read_state.SetIndices(info.block_idx, info.base_idx);
55150
+ read_state.PinData(sorted_data);
55151
+ const auto data_ptr = read_state.DataPtr(sorted_data);
55152
+ const auto next = vcount - info.base_idx;
55153
+
55154
+ // Set up a batch of pointers to scan data from
55155
+ Vector addresses(LogicalType::POINTER, next);
55156
+ auto data_pointers = FlatVector::GetData<data_ptr_t>(addresses);
55157
+
55158
+ // Set up the data pointers
55159
+ data_ptr_t row_ptr = data_ptr;
55160
+ const idx_t &row_width = sorted_data.layout.GetRowWidth();
55161
+ for (idx_t i = 0; i < next; ++i) {
55162
+ data_pointers[i] = row_ptr;
55163
+ row_ptr += row_width;
55164
+ }
55165
+ // Unswizzle the offsets back to pointers (if needed)
55166
+ if (!sorted_data.layout.AllConstant() && info.state.external) {
55167
+ RowOperations::UnswizzlePointers(sorted_data.layout, data_ptr, read_state.payload_heap_handle->Ptr(), next);
55168
+ }
55169
+
55170
+ // Deserialize the payload data
55171
+ for (idx_t col_idx = 0; col_idx < sorted_data.layout.ColumnCount(); col_idx++) {
55172
+ const auto col_offset = sorted_data.layout.GetOffsets()[col_idx];
55173
+ RowOperations::Gather(addresses, info.result, payload.data[left_cols + col_idx],
55174
+ *FlatVector::IncrementalSelectionVector(), result_count, col_offset, col_idx);
55175
+ }
55176
+ }
55177
+
55178
+ static void MergeJoinPinSortingBlock(SBScanState &scan, const idx_t block_idx) {
55179
+ scan.SetIndices(block_idx, 0);
55180
+ scan.PinRadix(block_idx);
55181
+
55182
+ auto &sd = *scan.sb->blob_sorting_data;
55183
+ if (block_idx < sd.data_blocks.size()) {
55184
+ scan.PinData(sd);
55185
+ }
55186
+ }
55187
+
55188
+ static data_ptr_t MergeJoinRadixPtr(SBScanState &scan, const idx_t entry_idx) {
55189
+ scan.entry_idx = entry_idx;
55190
+ return scan.RadixPtr();
55191
+ }
55192
+
55193
+ static idx_t MergeJoinSimpleBlocks(PiecewiseMergeJoinState &lstate, MergeJoinGlobalState &rstate, bool *found_match,
55194
+ const ExpressionType comparison) {
55195
+ const auto cmp = MergeJoinComparisonValue(comparison);
55196
+
55197
+ // The sort parameters should all be the same
55198
+ auto &lsort = *lstate.lhs_global_state;
55199
+ auto &rsort = rstate.rhs_global_sort_state;
55200
+ D_ASSERT(lsort.sort_layout.all_constant == rsort.sort_layout.all_constant);
55201
+ const auto all_constant = lsort.sort_layout.all_constant;
55202
+ D_ASSERT(lsort.external == rsort.external);
55203
+ const auto external = lsort.external;
55204
+
55205
+ // There should only be one sorted block if they have been sorted
55206
+ D_ASSERT(lsort.sorted_blocks.size() == 1);
55207
+ SBScanState lread(lsort.buffer_manager, lsort);
55208
+ lread.sb = lsort.sorted_blocks[0].get();
55209
+
55210
+ const idx_t l_block_idx = 0;
55211
+ idx_t l_entry_idx = 0;
55212
+ const auto lhs_not_null = lstate.lhs_count - lstate.lhs_has_null;
55213
+ MergeJoinPinSortingBlock(lread, l_block_idx);
55214
+ auto l_ptr = MergeJoinRadixPtr(lread, l_entry_idx);
55215
+
55216
+ D_ASSERT(rsort.sorted_blocks.size() == 1);
55217
+ SBScanState rread(rsort.buffer_manager, rsort);
55218
+ rread.sb = rsort.sorted_blocks[0].get();
55219
+
55220
+ const auto cmp_size = lsort.sort_layout.comparison_size;
55221
+ const auto entry_size = lsort.sort_layout.entry_size;
55222
+
55223
+ idx_t right_base = 0;
55224
+ for (idx_t r_block_idx = 0; r_block_idx < rread.sb->radix_sorting_data.size(); r_block_idx++) {
55225
+ // we only care about the BIGGEST value in each of the RHS data blocks
55226
+ // because we want to figure out if the LHS values are less than [or equal] to ANY value
55227
+ // get the biggest value from the RHS chunk
55228
+ MergeJoinPinSortingBlock(rread, r_block_idx);
55229
+
55230
+ auto &rblock = rread.sb->radix_sorting_data[r_block_idx];
55231
+ const auto r_not_null = SortedBlockNotNull(right_base, rblock.count, rstate.rhs_count - rstate.rhs_has_null);
55232
+ if (r_not_null == 0) {
55233
+ break;
55234
+ }
55235
+ const auto r_entry_idx = r_not_null - 1;
55236
+ right_base += rblock.count;
55237
+
55238
+ auto r_ptr = MergeJoinRadixPtr(rread, r_entry_idx);
55239
+
55240
+ // now we start from the current lpos value and check if we found a new value that is [<= OR <] the max RHS
55241
+ // value
55242
+ while (true) {
55243
+ int comp_res;
55244
+ if (all_constant) {
55245
+ comp_res = FastMemcmp(l_ptr, r_ptr, cmp_size);
55246
+ } else {
55247
+ lread.entry_idx = l_entry_idx;
55248
+ rread.entry_idx = r_entry_idx;
55249
+ comp_res = Comparators::CompareTuple(lread, rread, l_ptr, r_ptr, lsort.sort_layout, external);
55250
+ }
55251
+
55252
+ if (comp_res <= cmp) {
55253
+ // found a match for lpos, set it in the found_match vector
55254
+ found_match[l_entry_idx] = true;
55255
+ l_entry_idx++;
55256
+ l_ptr += entry_size;
55257
+ if (l_entry_idx >= lhs_not_null) {
55258
+ // early out: we exhausted the entire LHS and they all match
55259
+ return 0;
55260
+ }
55261
+ } else {
55262
+ // we found no match: any subsequent value from the LHS we scan now will be bigger and thus also not
55263
+ // match move to the next RHS chunk
55264
+ break;
55265
+ }
55266
+ }
55267
+ }
55268
+ return 0;
55269
55269
  }
55270
55270
 
55271
55271
  void PhysicalPiecewiseMergeJoin::ResolveSimpleJoin(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
@@ -55275,37 +55275,138 @@ void PhysicalPiecewiseMergeJoin::ResolveSimpleJoin(ExecutionContext &context, Da
55275
55275
 
55276
55276
  state.ResolveJoinKeys(input);
55277
55277
 
55278
- ScalarMergeInfo left_info(state.left_orders, state.join_keys.data[0].GetType(), state.left_position);
55279
- ChunkMergeInfo right_info(gstate.right_conditions, gstate.right_orders);
55280
-
55281
55278
  // perform the actual join
55282
- MergeJoinSimple::Perform(left_info, right_info, conditions[0].comparison);
55279
+ bool found_match[STANDARD_VECTOR_SIZE];
55280
+ memset(found_match, 0, sizeof(found_match));
55281
+ MergeJoinSimpleBlocks(state, gstate, found_match, conditions[0].comparison);
55283
55282
 
55284
- // now construct the result based ont he join result
55283
+ // extract the sorted payload
55284
+ idx_t left_position = 0;
55285
+ const auto lhs_not_null = state.lhs_count - state.lhs_has_null;
55286
+ BlockMergeInfo left_info(*state.lhs_global_state, 0, 0, left_position, lhs_not_null);
55287
+ left_info.result.Initialize(nullptr);
55288
+
55289
+ DataChunk payload;
55290
+ payload.Initialize(input.GetTypes());
55291
+ SliceSortedPayload(payload, left_info, state.lhs_count, state.lhs_count);
55292
+ payload.SetCardinality(state.lhs_count);
55293
+
55294
+ // now construct the result based on the join result
55285
55295
  switch (join_type) {
55286
- case JoinType::MARK:
55287
- PhysicalJoin::ConstructMarkJoinResult(state.join_keys, input, chunk, right_info.found_match, gstate.has_null);
55296
+ case JoinType::MARK: {
55297
+ // The only part of the join keys that is actually used is the validity mask.
55298
+ // Since the payload is sorted, we can just set the tail end of the validity masks to invalid.
55299
+ for (auto &key : state.lhs_keys.data) {
55300
+ key.Normalify(state.lhs_keys.size());
55301
+ auto &mask = FlatVector::Validity(key);
55302
+ if (mask.AllValid()) {
55303
+ continue;
55304
+ }
55305
+ mask.SetAllValid(lhs_not_null);
55306
+ for (idx_t i = lhs_not_null; i < state.lhs_count; ++i) {
55307
+ mask.SetInvalid(i);
55308
+ }
55309
+ }
55310
+ // So we make a set of keys that have the validity mask set for the
55311
+ PhysicalJoin::ConstructMarkJoinResult(state.lhs_keys, payload, chunk, found_match, gstate.rhs_has_null);
55288
55312
  break;
55313
+ }
55289
55314
  case JoinType::SEMI:
55290
- PhysicalJoin::ConstructSemiJoinResult(input, chunk, right_info.found_match);
55315
+ PhysicalJoin::ConstructSemiJoinResult(payload, chunk, found_match);
55291
55316
  break;
55292
55317
  case JoinType::ANTI:
55293
- PhysicalJoin::ConstructAntiJoinResult(input, chunk, right_info.found_match);
55318
+ PhysicalJoin::ConstructAntiJoinResult(payload, chunk, found_match);
55294
55319
  break;
55295
55320
  default:
55296
55321
  throw NotImplementedException("Unimplemented join type for merge join");
55297
55322
  }
55298
55323
  }
55299
55324
 
55325
+ static idx_t MergeJoinComplexBlocks(BlockMergeInfo &l, BlockMergeInfo &r, const ExpressionType comparison) {
55326
+ const auto cmp = MergeJoinComparisonValue(comparison);
55327
+
55328
+ // The sort parameters should all be the same
55329
+ D_ASSERT(l.state.sort_layout.all_constant == r.state.sort_layout.all_constant);
55330
+ const auto all_constant = r.state.sort_layout.all_constant;
55331
+ D_ASSERT(l.state.external == r.state.external);
55332
+ const auto external = l.state.external;
55333
+
55334
+ // There should only be one sorted block if they have been sorted
55335
+ D_ASSERT(l.state.sorted_blocks.size() == 1);
55336
+ SBScanState lread(l.state.buffer_manager, l.state);
55337
+ lread.sb = l.state.sorted_blocks[0].get();
55338
+ D_ASSERT(lread.sb->radix_sorting_data.size() == 1);
55339
+ MergeJoinPinSortingBlock(lread, l.block_idx);
55340
+ auto l_start = MergeJoinRadixPtr(lread, 0);
55341
+ auto l_ptr = MergeJoinRadixPtr(lread, l.entry_idx);
55342
+
55343
+ D_ASSERT(r.state.sorted_blocks.size() == 1);
55344
+ SBScanState rread(r.state.buffer_manager, r.state);
55345
+ rread.sb = r.state.sorted_blocks[0].get();
55346
+
55347
+ if (r.entry_idx >= r.not_null) {
55348
+ return 0;
55349
+ }
55350
+
55351
+ MergeJoinPinSortingBlock(rread, r.block_idx);
55352
+ auto r_ptr = MergeJoinRadixPtr(rread, r.entry_idx);
55353
+
55354
+ const auto cmp_size = l.state.sort_layout.comparison_size;
55355
+ const auto entry_size = l.state.sort_layout.entry_size;
55356
+
55357
+ idx_t result_count = 0;
55358
+ while (true) {
55359
+ if (l.entry_idx < l.not_null) {
55360
+ int comp_res;
55361
+ if (all_constant) {
55362
+ comp_res = FastMemcmp(l_ptr, r_ptr, cmp_size);
55363
+ } else {
55364
+ lread.entry_idx = l.entry_idx;
55365
+ rread.entry_idx = r.entry_idx;
55366
+ comp_res = Comparators::CompareTuple(lread, rread, l_ptr, r_ptr, l.state.sort_layout, external);
55367
+ }
55368
+
55369
+ if (comp_res <= cmp) {
55370
+ // left side smaller: found match
55371
+ l.result.set_index(result_count, sel_t(l.entry_idx - l.base_idx));
55372
+ r.result.set_index(result_count, sel_t(r.entry_idx - r.base_idx));
55373
+ result_count++;
55374
+ // move left side forward
55375
+ l.entry_idx++;
55376
+ l_ptr += entry_size;
55377
+ if (result_count == STANDARD_VECTOR_SIZE) {
55378
+ // out of space!
55379
+ break;
55380
+ }
55381
+ continue;
55382
+ }
55383
+ }
55384
+ // right side smaller or equal, or left side exhausted: move
55385
+ // right pointer forward reset left side to start
55386
+ r.entry_idx++;
55387
+ if (r.entry_idx >= r.not_null) {
55388
+ break;
55389
+ }
55390
+ r_ptr += entry_size;
55391
+
55392
+ l_ptr = l_start;
55393
+ l.entry_idx = 0;
55394
+ }
55395
+
55396
+ return result_count;
55397
+ }
55398
+
55300
55399
  OperatorResultType PhysicalPiecewiseMergeJoin::ResolveComplexJoin(ExecutionContext &context, DataChunk &input,
55301
55400
  DataChunk &chunk, OperatorState &state_p) const {
55302
55401
  auto &state = (PiecewiseMergeJoinState &)state_p;
55303
55402
  auto &gstate = (MergeJoinGlobalState &)*sink_state;
55403
+ auto &rsorted = *gstate.rhs_global_sort_state.sorted_blocks[0];
55304
55404
  do {
55305
55405
  if (state.first_fetch) {
55306
55406
  state.ResolveJoinKeys(input);
55307
55407
 
55308
55408
  state.right_chunk_index = 0;
55409
+ state.right_base = 0;
55309
55410
  state.left_position = 0;
55310
55411
  state.right_position = 0;
55311
55412
  state.first_fetch = false;
@@ -55315,47 +55416,53 @@ OperatorResultType PhysicalPiecewiseMergeJoin::ResolveComplexJoin(ExecutionConte
55315
55416
  if (IsLeftOuterJoin(join_type)) {
55316
55417
  // left join: before we move to the next chunk, see if we need to output any vectors that didn't
55317
55418
  // have a match found
55318
- PhysicalJoin::ConstructLeftJoinResult(input, chunk, state.left_found_match.get());
55319
- memset(state.left_found_match.get(), 0, sizeof(bool) * STANDARD_VECTOR_SIZE);
55419
+ PhysicalJoin::ConstructLeftJoinResult(input, chunk, state.lhs_found_match.get());
55420
+ memset(state.lhs_found_match.get(), 0, sizeof(bool) * STANDARD_VECTOR_SIZE);
55320
55421
  }
55321
55422
  state.first_fetch = true;
55322
55423
  state.finished = false;
55323
55424
  return OperatorResultType::NEED_MORE_INPUT;
55324
55425
  }
55325
55426
 
55326
- auto &right_chunk = gstate.right_chunks.GetChunk(state.right_chunk_index);
55327
- auto &right_condition_chunk = gstate.right_conditions.GetChunk(state.right_chunk_index);
55328
- auto &right_orders = gstate.right_orders[state.right_chunk_index];
55427
+ const auto lhs_not_null = state.lhs_count - state.lhs_has_null;
55428
+ BlockMergeInfo left_info(*state.lhs_global_state, 0, 0, state.left_position, lhs_not_null);
55329
55429
 
55330
- ScalarMergeInfo left_info(state.left_orders, state.join_keys.data[0].GetType(), state.left_position);
55331
- ScalarMergeInfo right_info(right_orders, right_condition_chunk.data[0].GetType(), state.right_position);
55430
+ const auto &rblock = rsorted.radix_sorting_data[state.right_chunk_index];
55431
+ const auto rhs_not_null =
55432
+ SortedBlockNotNull(state.right_base, rblock.count, gstate.rhs_count - gstate.rhs_has_null);
55433
+ BlockMergeInfo right_info(gstate.rhs_global_sort_state, state.right_chunk_index, state.right_position,
55434
+ state.right_position, rhs_not_null);
55332
55435
 
55333
- idx_t result_count = MergeJoinComplex::Perform(left_info, right_info, conditions[0].comparison);
55436
+ idx_t result_count = MergeJoinComplexBlocks(left_info, right_info, conditions[0].comparison);
55334
55437
  if (result_count == 0) {
55335
55438
  // exhausted this chunk on the right side
55336
55439
  // move to the next right chunk
55337
55440
  state.left_position = 0;
55338
55441
  state.right_position = 0;
55442
+ state.right_base += rsorted.radix_sorting_data[state.right_chunk_index].count;
55339
55443
  state.right_chunk_index++;
55340
- if (state.right_chunk_index >= gstate.right_chunks.ChunkCount()) {
55444
+ if (state.right_chunk_index >= rsorted.radix_sorting_data.size()) {
55341
55445
  state.finished = true;
55342
55446
  }
55343
55447
  } else {
55344
55448
  // found matches: mark the found matches if required
55345
- if (state.left_found_match) {
55449
+ if (state.lhs_found_match) {
55346
55450
  for (idx_t i = 0; i < result_count; i++) {
55347
- state.left_found_match[left_info.result.get_index(i)] = true;
55451
+ state.lhs_found_match[left_info.result[i]] = true;
55348
55452
  }
55349
55453
  }
55350
- if (gstate.right_found_match) {
55351
- idx_t base_index = state.right_chunk_index * STANDARD_VECTOR_SIZE;
55454
+ if (gstate.rhs_found_match) {
55455
+ // Absolute position of the block + start position inside that block
55456
+ const idx_t base_index = state.right_base + right_info.base_idx;
55352
55457
  for (idx_t i = 0; i < result_count; i++) {
55353
- gstate.right_found_match[base_index + right_info.result.get_index(i)] = true;
55458
+ gstate.rhs_found_match[base_index + right_info.result[i]] = true;
55354
55459
  }
55355
55460
  }
55356
55461
  // found matches: output them
55357
- chunk.Slice(input, left_info.result, result_count);
55358
- chunk.Slice(right_chunk, right_info.result, result_count, input.ColumnCount());
55462
+ SliceSortedPayload(chunk, left_info, result_count, input.size());
55463
+ SliceSortedPayload(chunk, right_info, result_count, right_info.entry_idx + 1, input.ColumnCount());
55464
+ chunk.SetCardinality(result_count);
55465
+ chunk.Verify();
55359
55466
  }
55360
55467
  } while (chunk.size() == 0);
55361
55468
  return OperatorResultType::HAVE_MORE_OUTPUT;
@@ -55365,10 +55472,10 @@ OperatorResultType PhysicalPiecewiseMergeJoin::Execute(ExecutionContext &context
55365
55472
  OperatorState &state) const {
55366
55473
  auto &gstate = (MergeJoinGlobalState &)*sink_state;
55367
55474
 
55368
- if (gstate.right_chunks.Count() == 0) {
55475
+ if (gstate.Count() == 0) {
55369
55476
  // empty RHS
55370
55477
  if (!EmptyResultIfRHSIsEmpty()) {
55371
- ConstructEmptyJoinResult(join_type, gstate.has_null, input, chunk);
55478
+ ConstructEmptyJoinResult(join_type, gstate.rhs_has_null, input, chunk);
55372
55479
  return OperatorResultType::NEED_MORE_INPUT;
55373
55480
  } else {
55374
55481
  return OperatorResultType::FINISHED;
@@ -55392,169 +55499,6 @@ OperatorResultType PhysicalPiecewiseMergeJoin::Execute(ExecutionContext &context
55392
55499
  }
55393
55500
  }
55394
55501
 
55395
- //===--------------------------------------------------------------------===//
55396
- // OrderVector
55397
- //===--------------------------------------------------------------------===//
55398
- template <class T, class OP>
55399
- static sel_t TemplatedQuicksortInitial(T *data, const SelectionVector &sel, const SelectionVector &not_null_sel,
55400
- idx_t count, SelectionVector &result) {
55401
- // select pivot
55402
- auto pivot_idx = not_null_sel.get_index(0);
55403
- auto dpivot_idx = sel.get_index(pivot_idx);
55404
- sel_t low = 0, high = count - 1;
55405
- // now insert elements
55406
- for (idx_t i = 1; i < count; i++) {
55407
- auto idx = not_null_sel.get_index(i);
55408
- auto didx = sel.get_index(idx);
55409
- if (OP::Operation(data[didx], data[dpivot_idx])) {
55410
- result.set_index(low++, idx);
55411
- } else {
55412
- result.set_index(high--, idx);
55413
- }
55414
- }
55415
- D_ASSERT(low == high);
55416
- result.set_index(low, pivot_idx);
55417
- return low;
55418
- }
55419
-
55420
- struct QuickSortPivot {
55421
- QuickSortPivot(sel_t left_p, sel_t right_p) : left(left_p), right(right_p) {
55422
- }
55423
-
55424
- sel_t left;
55425
- sel_t right;
55426
- };
55427
-
55428
- template <class T, class OP>
55429
- static void TemplatedQuicksortRefine(T *data, const SelectionVector &sel, idx_t count, SelectionVector &result,
55430
- sel_t left, sel_t right, vector<QuickSortPivot> &pivots) {
55431
- if (left >= right) {
55432
- return;
55433
- }
55434
-
55435
- sel_t middle = left + (right - left) / 2;
55436
- sel_t dpivot_idx = sel.get_index(result.get_index(middle));
55437
-
55438
- // move the mid point value to the front.
55439
- sel_t i = left + 1;
55440
- sel_t j = right;
55441
-
55442
- result.swap(middle, left);
55443
- while (i <= j) {
55444
- while (i <= j && (OP::Operation(data[sel.get_index(result.get_index(i))], data[dpivot_idx]))) {
55445
- i++;
55446
- }
55447
-
55448
- while (i <= j && !OP::Operation(data[sel.get_index(result.get_index(j))], data[dpivot_idx])) {
55449
- j--;
55450
- }
55451
-
55452
- if (i < j) {
55453
- result.swap(i, j);
55454
- }
55455
- }
55456
- result.swap(i - 1, left);
55457
- sel_t part = i - 1;
55458
-
55459
- if (part > 0) {
55460
- pivots.emplace_back(left, part - 1);
55461
- }
55462
- if (part + 1 < right) {
55463
- pivots.emplace_back(part + 1, right);
55464
- }
55465
- }
55466
-
55467
- template <class T, class OP>
55468
- void TemplatedQuicksort(T *__restrict data, const SelectionVector &sel, const SelectionVector &not_null_sel,
55469
- idx_t count, SelectionVector &result) {
55470
- auto part = TemplatedQuicksortInitial<T, OP>(data, sel, not_null_sel, count, result);
55471
- if (part > count) {
55472
- return;
55473
- }
55474
- vector<QuickSortPivot> pivots;
55475
- pivots.emplace_back(0, part);
55476
- pivots.emplace_back(part + 1, count - 1);
55477
- for (idx_t i = 0; i < pivots.size(); i++) {
55478
- auto pivot = pivots[i];
55479
- TemplatedQuicksortRefine<T, OP>(data, sel, count, result, pivot.left, pivot.right, pivots);
55480
- }
55481
- }
55482
-
55483
- template <class T>
55484
- static void TemplatedQuicksort(VectorData &vdata, const SelectionVector &not_null_sel, idx_t not_null_count,
55485
- SelectionVector &result) {
55486
- if (not_null_count == 0) {
55487
- return;
55488
- }
55489
- TemplatedQuicksort<T, duckdb::LessThanEquals>((T *)vdata.data, *vdata.sel, not_null_sel, not_null_count, result);
55490
- }
55491
-
55492
- void OrderVector(Vector &vector, idx_t count, MergeOrder &order) {
55493
- if (count == 0) {
55494
- order.count = 0;
55495
- return;
55496
- }
55497
- vector.Orrify(count, order.vdata);
55498
- auto &vdata = order.vdata;
55499
-
55500
- // first filter out all the non-null values
55501
- SelectionVector not_null(STANDARD_VECTOR_SIZE);
55502
- idx_t not_null_count = 0;
55503
- for (idx_t i = 0; i < count; i++) {
55504
- auto idx = vdata.sel->get_index(i);
55505
- if (vdata.validity.RowIsValid(idx)) {
55506
- not_null.set_index(not_null_count++, i);
55507
- }
55508
- }
55509
-
55510
- order.count = not_null_count;
55511
- order.order.Initialize(STANDARD_VECTOR_SIZE);
55512
- switch (vector.GetType().InternalType()) {
55513
- case PhysicalType::BOOL:
55514
- case PhysicalType::INT8:
55515
- TemplatedQuicksort<int8_t>(vdata, not_null, not_null_count, order.order);
55516
- break;
55517
- case PhysicalType::INT16:
55518
- TemplatedQuicksort<int16_t>(vdata, not_null, not_null_count, order.order);
55519
- break;
55520
- case PhysicalType::INT32:
55521
- TemplatedQuicksort<int32_t>(vdata, not_null, not_null_count, order.order);
55522
- break;
55523
- case PhysicalType::INT64:
55524
- TemplatedQuicksort<int64_t>(vdata, not_null, not_null_count, order.order);
55525
- break;
55526
- case PhysicalType::UINT8:
55527
- TemplatedQuicksort<uint8_t>(vdata, not_null, not_null_count, order.order);
55528
- break;
55529
- case PhysicalType::UINT16:
55530
- TemplatedQuicksort<uint16_t>(vdata, not_null, not_null_count, order.order);
55531
- break;
55532
- case PhysicalType::UINT32:
55533
- TemplatedQuicksort<uint32_t>(vdata, not_null, not_null_count, order.order);
55534
- break;
55535
- case PhysicalType::UINT64:
55536
- TemplatedQuicksort<uint64_t>(vdata, not_null, not_null_count, order.order);
55537
- break;
55538
- case PhysicalType::INT128:
55539
- TemplatedQuicksort<hugeint_t>(vdata, not_null, not_null_count, order.order);
55540
- break;
55541
- case PhysicalType::FLOAT:
55542
- TemplatedQuicksort<float>(vdata, not_null, not_null_count, order.order);
55543
- break;
55544
- case PhysicalType::DOUBLE:
55545
- TemplatedQuicksort<double>(vdata, not_null, not_null_count, order.order);
55546
- break;
55547
- case PhysicalType::INTERVAL:
55548
- TemplatedQuicksort<interval_t>(vdata, not_null, not_null_count, order.order);
55549
- break;
55550
- case PhysicalType::VARCHAR:
55551
- TemplatedQuicksort<string_t>(vdata, not_null, not_null_count, order.order);
55552
- break;
55553
- default:
55554
- throw NotImplementedException("Unimplemented type for sort");
55555
- }
55556
- }
55557
-
55558
55502
  //===--------------------------------------------------------------------===//
55559
55503
  // Source
55560
55504
  //===--------------------------------------------------------------------===//
@@ -55565,12 +55509,13 @@ public:
55565
55509
 
55566
55510
  mutex lock;
55567
55511
  const PhysicalPiecewiseMergeJoin &op;
55512
+ unique_ptr<PayloadScanner> scanner;
55568
55513
  idx_t right_outer_position;
55569
55514
 
55570
55515
  public:
55571
55516
  idx_t MaxThreads() override {
55572
55517
  auto &sink = (MergeJoinGlobalState &)*op.sink_state;
55573
- return sink.right_chunks.Count() / (STANDARD_VECTOR_SIZE * 10);
55518
+ return sink.Count() / (STANDARD_VECTOR_SIZE * idx_t(10));
55574
55519
  }
55575
55520
  };
55576
55521
 
@@ -55578,17 +55523,63 @@ unique_ptr<GlobalSourceState> PhysicalPiecewiseMergeJoin::GetGlobalSourceState(C
55578
55523
  return make_unique<PiecewiseJoinScanState>(*this);
55579
55524
  }
55580
55525
 
55581
- void PhysicalPiecewiseMergeJoin::GetData(ExecutionContext &context, DataChunk &chunk, GlobalSourceState &gstate,
55526
+ void PhysicalPiecewiseMergeJoin::GetData(ExecutionContext &context, DataChunk &result, GlobalSourceState &gstate,
55582
55527
  LocalSourceState &lstate) const {
55583
55528
  D_ASSERT(IsRightOuterJoin(join_type));
55584
55529
  // check if we need to scan any unmatched tuples from the RHS for the full/right outer join
55585
55530
  auto &sink = (MergeJoinGlobalState &)*sink_state;
55586
55531
  auto &state = (PiecewiseJoinScanState &)gstate;
55587
55532
 
55533
+ lock_guard<mutex> l(state.lock);
55534
+ if (!state.scanner) {
55535
+ // Initialize scanner (if not yet initialized)
55536
+ auto &sort_state = sink.rhs_global_sort_state;
55537
+ if (sort_state.sorted_blocks.empty()) {
55538
+ return;
55539
+ }
55540
+ state.scanner = make_unique<PayloadScanner>(*sort_state.sorted_blocks[0]->payload_data, sort_state);
55541
+ }
55542
+
55588
55543
  // if the LHS is exhausted in a FULL/RIGHT OUTER JOIN, we scan the found_match for any chunks we
55589
55544
  // still need to output
55590
- lock_guard<mutex> l(state.lock);
55591
- ConstructFullOuterJoinResult(sink.right_found_match.get(), sink.right_chunks, chunk, state.right_outer_position);
55545
+ const auto found_match = sink.rhs_found_match.get();
55546
+
55547
+ // ConstructFullOuterJoinResult(sink.rhs_found_match.get(), sink.right_chunks, chunk, state.right_outer_position);
55548
+ DataChunk rhs_chunk;
55549
+ rhs_chunk.Initialize(children[1]->types);
55550
+ SelectionVector rsel(STANDARD_VECTOR_SIZE);
55551
+ for (;;) {
55552
+ // Read the next sorted chunk
55553
+ state.scanner->Scan(rhs_chunk);
55554
+
55555
+ const auto count = rhs_chunk.size();
55556
+ if (count == 0) {
55557
+ return;
55558
+ }
55559
+
55560
+ idx_t result_count = 0;
55561
+ // figure out which tuples didn't find a match in the RHS
55562
+ for (idx_t i = 0; i < count; i++) {
55563
+ if (!found_match[state.right_outer_position + i]) {
55564
+ rsel.set_index(result_count++, i);
55565
+ }
55566
+ }
55567
+ state.right_outer_position += count;
55568
+
55569
+ if (result_count > 0) {
55570
+ // if there were any tuples that didn't find a match, output them
55571
+ const idx_t left_column_count = result.ColumnCount() - rhs_chunk.ColumnCount();
55572
+ for (idx_t col_idx = 0; col_idx < left_column_count; ++col_idx) {
55573
+ result.data[col_idx].SetVectorType(VectorType::CONSTANT_VECTOR);
55574
+ ConstantVector::SetNull(result.data[col_idx], true);
55575
+ }
55576
+ for (idx_t col_idx = 0; col_idx < rhs_chunk.ColumnCount(); ++col_idx) {
55577
+ result.data[left_column_count + col_idx].Slice(rhs_chunk.data[col_idx], rsel, result_count);
55578
+ }
55579
+ result.SetCardinality(result_count);
55580
+ return;
55581
+ }
55582
+ }
55592
55583
  }
55593
55584
 
55594
55585
  } // namespace duckdb
@@ -57358,7 +57349,7 @@ void BufferedCSVReader::DetectHeader(const vector<vector<LogicalType>> &best_sql
57358
57349
  // update parser info, and read, generate & set col_names based on previous findings
57359
57350
  if (((!first_row_consistent || first_row_nulls) && !options.has_header) || (options.has_header && options.header)) {
57360
57351
  options.header = true;
57361
- unordered_map<string, idx_t> name_collision_count;
57352
+ case_insensitive_map_t<idx_t> name_collision_count;
57362
57353
  // get header names from CSV
57363
57354
  for (idx_t col = 0; col < options.num_cols; col++) {
57364
57355
  const auto &val = best_header_row.GetValue(col, 0);
@@ -74230,7 +74221,7 @@ string Function::CallToString(const string &name, const vector<LogicalType> &arg
74230
74221
  }
74231
74222
 
74232
74223
  string Function::CallToString(const string &name, const vector<LogicalType> &arguments,
74233
- const unordered_map<string, LogicalType> &named_parameters) {
74224
+ const named_parameter_type_map_t &named_parameters) {
74234
74225
  vector<string> input_arguments;
74235
74226
  input_arguments.reserve(arguments.size() + named_parameters.size());
74236
74227
  for (auto &arg : arguments) {
@@ -79996,7 +79987,7 @@ bool ClampIndex(INDEX_TYPE &index, const INPUT_TYPE &value) {
79996
79987
  }
79997
79988
  index = length + index;
79998
79989
  } else if (index > length) {
79999
- return false;
79990
+ index = length;
80000
79991
  }
80001
79992
  return true;
80002
79993
  }
@@ -89613,7 +89604,7 @@ public:
89613
89604
  private:
89614
89605
  //! Binds an arrow table
89615
89606
  static unique_ptr<FunctionData> ArrowScanBind(ClientContext &context, vector<Value> &inputs,
89616
- unordered_map<string, Value> &named_parameters,
89607
+ named_parameter_map_t &named_parameters,
89617
89608
  vector<LogicalType> &input_table_types,
89618
89609
  vector<string> &input_table_names, vector<LogicalType> &return_types,
89619
89610
  vector<string> &names);
@@ -89813,7 +89804,7 @@ LogicalType GetArrowLogicalType(ArrowSchema &schema,
89813
89804
  }
89814
89805
 
89815
89806
  unique_ptr<FunctionData> ArrowTableFunction::ArrowScanBind(ClientContext &context, vector<Value> &inputs,
89816
- unordered_map<string, Value> &named_parameters,
89807
+ named_parameter_map_t &named_parameters,
89817
89808
  vector<LogicalType> &input_table_types,
89818
89809
  vector<string> &input_table_names,
89819
89810
  vector<LogicalType> &return_types, vector<string> &names) {
@@ -90817,7 +90808,7 @@ struct UnnestTableFunction {
90817
90808
  namespace duckdb {
90818
90809
 
90819
90810
  static unique_ptr<FunctionData> CheckpointBind(ClientContext &context, vector<Value> &inputs,
90820
- unordered_map<string, Value> &named_parameters,
90811
+ named_parameter_map_t &named_parameters,
90821
90812
  vector<LogicalType> &input_table_types,
90822
90813
  vector<string> &input_table_names, vector<LogicalType> &return_types,
90823
90814
  vector<string> &names) {
@@ -91469,7 +91460,7 @@ struct GlobFunctionBindData : public TableFunctionData {
91469
91460
  };
91470
91461
 
91471
91462
  static unique_ptr<FunctionData> GlobFunctionBind(ClientContext &context, vector<Value> &inputs,
91472
- unordered_map<string, Value> &named_parameters,
91463
+ named_parameter_map_t &named_parameters,
91473
91464
  vector<LogicalType> &input_table_types,
91474
91465
  vector<string> &input_table_names, vector<LogicalType> &return_types,
91475
91466
  vector<string> &names) {
@@ -91646,7 +91637,7 @@ struct PragmaDetailedProfilingOutputData : public TableFunctionData {
91646
91637
  };
91647
91638
 
91648
91639
  static unique_ptr<FunctionData> PragmaDetailedProfilingOutputBind(ClientContext &context, vector<Value> &inputs,
91649
- unordered_map<string, Value> &named_parameters,
91640
+ named_parameter_map_t &named_parameters,
91650
91641
  vector<LogicalType> &input_table_types,
91651
91642
  vector<string> &input_table_names,
91652
91643
  vector<LogicalType> &return_types,
@@ -91822,12 +91813,10 @@ struct PragmaLastProfilingOutputData : public TableFunctionData {
91822
91813
  vector<LogicalType> types;
91823
91814
  };
91824
91815
 
91825
- static unique_ptr<FunctionData> PragmaLastProfilingOutputBind(ClientContext &context, vector<Value> &inputs,
91826
- unordered_map<string, Value> &named_parameters,
91827
- vector<LogicalType> &input_table_types,
91828
- vector<string> &input_table_names,
91829
- vector<LogicalType> &return_types,
91830
- vector<string> &names) {
91816
+ static unique_ptr<FunctionData>
91817
+ PragmaLastProfilingOutputBind(ClientContext &context, vector<Value> &inputs, named_parameter_map_t &named_parameters,
91818
+ vector<LogicalType> &input_table_types, vector<string> &input_table_names,
91819
+ vector<LogicalType> &return_types, vector<string> &names) {
91831
91820
  names.emplace_back("OPERATOR_ID");
91832
91821
  return_types.emplace_back(LogicalType::INTEGER);
91833
91822
 
@@ -91941,7 +91930,7 @@ struct RangeFunctionBindData : public TableFunctionData {
91941
91930
 
91942
91931
  template <bool GENERATE_SERIES>
91943
91932
  static unique_ptr<FunctionData>
91944
- RangeFunctionBind(ClientContext &context, vector<Value> &inputs, unordered_map<string, Value> &named_parameters,
91933
+ RangeFunctionBind(ClientContext &context, vector<Value> &inputs, named_parameter_map_t &named_parameters,
91945
91934
  vector<LogicalType> &input_table_types, vector<string> &input_table_names,
91946
91935
  vector<LogicalType> &return_types, vector<string> &names) {
91947
91936
  auto result = make_unique<RangeFunctionBindData>();
@@ -92052,7 +92041,7 @@ struct RangeDateTimeBindData : public TableFunctionData {
92052
92041
 
92053
92042
  template <bool GENERATE_SERIES>
92054
92043
  static unique_ptr<FunctionData>
92055
- RangeDateTimeBind(ClientContext &context, vector<Value> &inputs, unordered_map<string, Value> &named_parameters,
92044
+ RangeDateTimeBind(ClientContext &context, vector<Value> &inputs, named_parameter_map_t &named_parameters,
92056
92045
  vector<LogicalType> &input_table_types, vector<string> &input_table_names,
92057
92046
  vector<LogicalType> &return_types, vector<string> &names) {
92058
92047
  auto result = make_unique<RangeDateTimeBindData>();
@@ -92191,7 +92180,7 @@ void BuiltinFunctions::RegisterTableFunctions() {
92191
92180
  namespace duckdb {
92192
92181
 
92193
92182
  static unique_ptr<FunctionData> ReadCSVBind(ClientContext &context, vector<Value> &inputs,
92194
- unordered_map<string, Value> &named_parameters,
92183
+ named_parameter_map_t &named_parameters,
92195
92184
  vector<LogicalType> &input_table_types, vector<string> &input_table_names,
92196
92185
  vector<LogicalType> &return_types, vector<string> &names) {
92197
92186
  auto &config = DBConfig::GetConfig(context);
@@ -92210,22 +92199,23 @@ static unique_ptr<FunctionData> ReadCSVBind(ClientContext &context, vector<Value
92210
92199
  }
92211
92200
 
92212
92201
  for (auto &kv : named_parameters) {
92213
- if (kv.first == "auto_detect") {
92202
+ auto loption = StringUtil::Lower(kv.first);
92203
+ if (loption == "auto_detect") {
92214
92204
  options.auto_detect = BooleanValue::Get(kv.second);
92215
- } else if (kv.first == "sep" || kv.first == "delim") {
92205
+ } else if (loption == "sep" || loption == "delim") {
92216
92206
  options.SetDelimiter(StringValue::Get(kv.second));
92217
- } else if (kv.first == "header") {
92207
+ } else if (loption == "header") {
92218
92208
  options.header = BooleanValue::Get(kv.second);
92219
92209
  options.has_header = true;
92220
- } else if (kv.first == "quote") {
92210
+ } else if (loption == "quote") {
92221
92211
  options.quote = StringValue::Get(kv.second);
92222
92212
  options.has_quote = true;
92223
- } else if (kv.first == "escape") {
92213
+ } else if (loption == "escape") {
92224
92214
  options.escape = StringValue::Get(kv.second);
92225
92215
  options.has_escape = true;
92226
- } else if (kv.first == "nullstr") {
92216
+ } else if (loption == "nullstr") {
92227
92217
  options.null_str = StringValue::Get(kv.second);
92228
- } else if (kv.first == "sample_size") {
92218
+ } else if (loption == "sample_size") {
92229
92219
  int64_t sample_size = kv.second.GetValue<int64_t>();
92230
92220
  if (sample_size < 1 && sample_size != -1) {
92231
92221
  throw BinderException("Unsupported parameter for SAMPLE_SIZE: cannot be smaller than 1");
@@ -92240,7 +92230,7 @@ static unique_ptr<FunctionData> ReadCSVBind(ClientContext &context, vector<Value
92240
92230
  options.sample_chunk_size = STANDARD_VECTOR_SIZE;
92241
92231
  options.sample_chunks = sample_size / STANDARD_VECTOR_SIZE;
92242
92232
  }
92243
- } else if (kv.first == "sample_chunk_size") {
92233
+ } else if (loption == "sample_chunk_size") {
92244
92234
  options.sample_chunk_size = kv.second.GetValue<int64_t>();
92245
92235
  if (options.sample_chunk_size > STANDARD_VECTOR_SIZE) {
92246
92236
  throw BinderException(
@@ -92249,14 +92239,14 @@ static unique_ptr<FunctionData> ReadCSVBind(ClientContext &context, vector<Value
92249
92239
  } else if (options.sample_chunk_size < 1) {
92250
92240
  throw BinderException("Unsupported parameter for SAMPLE_CHUNK_SIZE: cannot be smaller than 1");
92251
92241
  }
92252
- } else if (kv.first == "sample_chunks") {
92242
+ } else if (loption == "sample_chunks") {
92253
92243
  options.sample_chunks = kv.second.GetValue<int64_t>();
92254
92244
  if (options.sample_chunks < 1) {
92255
92245
  throw BinderException("Unsupported parameter for SAMPLE_CHUNKS: cannot be smaller than 1");
92256
92246
  }
92257
- } else if (kv.first == "all_varchar") {
92247
+ } else if (loption == "all_varchar") {
92258
92248
  options.all_varchar = BooleanValue::Get(kv.second);
92259
- } else if (kv.first == "dateformat") {
92249
+ } else if (loption == "dateformat") {
92260
92250
  options.has_format[LogicalTypeId::DATE] = true;
92261
92251
  auto &date_format = options.date_format[LogicalTypeId::DATE];
92262
92252
  date_format.format_specifier = StringValue::Get(kv.second);
@@ -92264,7 +92254,7 @@ static unique_ptr<FunctionData> ReadCSVBind(ClientContext &context, vector<Value
92264
92254
  if (!error.empty()) {
92265
92255
  throw InvalidInputException("Could not parse DATEFORMAT: %s", error.c_str());
92266
92256
  }
92267
- } else if (kv.first == "timestampformat") {
92257
+ } else if (loption == "timestampformat") {
92268
92258
  options.has_format[LogicalTypeId::TIMESTAMP] = true;
92269
92259
  auto &timestamp_format = options.date_format[LogicalTypeId::TIMESTAMP];
92270
92260
  timestamp_format.format_specifier = StringValue::Get(kv.second);
@@ -92272,9 +92262,9 @@ static unique_ptr<FunctionData> ReadCSVBind(ClientContext &context, vector<Value
92272
92262
  if (!error.empty()) {
92273
92263
  throw InvalidInputException("Could not parse TIMESTAMPFORMAT: %s", error.c_str());
92274
92264
  }
92275
- } else if (kv.first == "normalize_names") {
92265
+ } else if (loption == "normalize_names") {
92276
92266
  options.normalize_names = BooleanValue::Get(kv.second);
92277
- } else if (kv.first == "columns") {
92267
+ } else if (loption == "columns") {
92278
92268
  auto &child_type = kv.second.type();
92279
92269
  if (child_type.id() != LogicalTypeId::STRUCT) {
92280
92270
  throw BinderException("read_csv columns requires a a struct as input");
@@ -92293,12 +92283,14 @@ static unique_ptr<FunctionData> ReadCSVBind(ClientContext &context, vector<Value
92293
92283
  if (names.empty()) {
92294
92284
  throw BinderException("read_csv requires at least a single column as input!");
92295
92285
  }
92296
- } else if (kv.first == "compression") {
92286
+ } else if (loption == "compression") {
92297
92287
  options.compression = FileCompressionTypeFromString(StringValue::Get(kv.second));
92298
- } else if (kv.first == "filename") {
92288
+ } else if (loption == "filename") {
92299
92289
  result->include_file_name = BooleanValue::Get(kv.second);
92300
- } else if (kv.first == "skip") {
92290
+ } else if (loption == "skip") {
92301
92291
  options.skip_rows = kv.second.GetValue<int64_t>();
92292
+ } else {
92293
+ throw InternalException("Unrecognized parameter %s", kv.first);
92302
92294
  }
92303
92295
  }
92304
92296
  if (!options.auto_detect && return_types.empty()) {
@@ -92352,7 +92344,7 @@ static unique_ptr<FunctionOperatorData> ReadCSVInit(ClientContext &context, cons
92352
92344
  }
92353
92345
 
92354
92346
  static unique_ptr<FunctionData> ReadCSVAutoBind(ClientContext &context, vector<Value> &inputs,
92355
- unordered_map<string, Value> &named_parameters,
92347
+ named_parameter_map_t &named_parameters,
92356
92348
  vector<LogicalType> &input_table_types,
92357
92349
  vector<string> &input_table_names, vector<LogicalType> &return_types,
92358
92350
  vector<string> &names) {
@@ -92477,7 +92469,7 @@ struct RepeatOperatorData : public FunctionOperatorData {
92477
92469
  };
92478
92470
 
92479
92471
  static unique_ptr<FunctionData> RepeatBind(ClientContext &context, vector<Value> &inputs,
92480
- unordered_map<string, Value> &named_parameters,
92472
+ named_parameter_map_t &named_parameters,
92481
92473
  vector<LogicalType> &input_table_types, vector<string> &input_table_names,
92482
92474
  vector<LogicalType> &return_types, vector<string> &names) {
92483
92475
  // the repeat function returns the type of the first argument
@@ -92524,7 +92516,7 @@ void RepeatTableFunction::RegisterFunction(BuiltinFunctions &set) {
92524
92516
  namespace duckdb {
92525
92517
 
92526
92518
  static unique_ptr<FunctionData> SummaryFunctionBind(ClientContext &context, vector<Value> &inputs,
92527
- unordered_map<string, Value> &named_parameters,
92519
+ named_parameter_map_t &named_parameters,
92528
92520
  vector<LogicalType> &input_table_types,
92529
92521
  vector<string> &input_table_names,
92530
92522
  vector<LogicalType> &return_types, vector<string> &names) {
@@ -92592,7 +92584,7 @@ struct DuckDBColumnsData : public FunctionOperatorData {
92592
92584
  };
92593
92585
 
92594
92586
  static unique_ptr<FunctionData> DuckDBColumnsBind(ClientContext &context, vector<Value> &inputs,
92595
- unordered_map<string, Value> &named_parameters,
92587
+ named_parameter_map_t &named_parameters,
92596
92588
  vector<LogicalType> &input_table_types,
92597
92589
  vector<string> &input_table_names, vector<LogicalType> &return_types,
92598
92590
  vector<string> &names) {
@@ -92927,7 +92919,7 @@ struct DuckDBConstraintsData : public FunctionOperatorData {
92927
92919
  };
92928
92920
 
92929
92921
  static unique_ptr<FunctionData> DuckDBConstraintsBind(ClientContext &context, vector<Value> &inputs,
92930
- unordered_map<string, Value> &named_parameters,
92922
+ named_parameter_map_t &named_parameters,
92931
92923
  vector<LogicalType> &input_table_types,
92932
92924
  vector<string> &input_table_names,
92933
92925
  vector<LogicalType> &return_types, vector<string> &names) {
@@ -93131,7 +93123,7 @@ struct DuckDBDependenciesData : public FunctionOperatorData {
93131
93123
  };
93132
93124
 
93133
93125
  static unique_ptr<FunctionData> DuckDBDependenciesBind(ClientContext &context, vector<Value> &inputs,
93134
- unordered_map<string, Value> &named_parameters,
93126
+ named_parameter_map_t &named_parameters,
93135
93127
  vector<LogicalType> &input_table_types,
93136
93128
  vector<string> &input_table_names,
93137
93129
  vector<LogicalType> &return_types, vector<string> &names) {
@@ -93254,7 +93246,7 @@ struct DuckDBFunctionsData : public FunctionOperatorData {
93254
93246
  };
93255
93247
 
93256
93248
  static unique_ptr<FunctionData> DuckDBFunctionsBind(ClientContext &context, vector<Value> &inputs,
93257
- unordered_map<string, Value> &named_parameters,
93249
+ named_parameter_map_t &named_parameters,
93258
93250
  vector<LogicalType> &input_table_types,
93259
93251
  vector<string> &input_table_names,
93260
93252
  vector<LogicalType> &return_types, vector<string> &names) {
@@ -93665,7 +93657,7 @@ struct DuckDBIndexesData : public FunctionOperatorData {
93665
93657
  };
93666
93658
 
93667
93659
  static unique_ptr<FunctionData> DuckDBIndexesBind(ClientContext &context, vector<Value> &inputs,
93668
- unordered_map<string, Value> &named_parameters,
93660
+ named_parameter_map_t &named_parameters,
93669
93661
  vector<LogicalType> &input_table_types,
93670
93662
  vector<string> &input_table_names, vector<LogicalType> &return_types,
93671
93663
  vector<string> &names) {
@@ -93785,7 +93777,7 @@ struct DuckDBKeywordsData : public FunctionOperatorData {
93785
93777
  };
93786
93778
 
93787
93779
  static unique_ptr<FunctionData> DuckDBKeywordsBind(ClientContext &context, vector<Value> &inputs,
93788
- unordered_map<string, Value> &named_parameters,
93780
+ named_parameter_map_t &named_parameters,
93789
93781
  vector<LogicalType> &input_table_types,
93790
93782
  vector<string> &input_table_names, vector<LogicalType> &return_types,
93791
93783
  vector<string> &names) {
@@ -93870,7 +93862,7 @@ struct DuckDBSchemasData : public FunctionOperatorData {
93870
93862
  };
93871
93863
 
93872
93864
  static unique_ptr<FunctionData> DuckDBSchemasBind(ClientContext &context, vector<Value> &inputs,
93873
- unordered_map<string, Value> &named_parameters,
93865
+ named_parameter_map_t &named_parameters,
93874
93866
  vector<LogicalType> &input_table_types,
93875
93867
  vector<string> &input_table_names, vector<LogicalType> &return_types,
93876
93868
  vector<string> &names) {
@@ -93955,7 +93947,7 @@ struct DuckDBSequencesData : public FunctionOperatorData {
93955
93947
  };
93956
93948
 
93957
93949
  static unique_ptr<FunctionData> DuckDBSequencesBind(ClientContext &context, vector<Value> &inputs,
93958
- unordered_map<string, Value> &named_parameters,
93950
+ named_parameter_map_t &named_parameters,
93959
93951
  vector<LogicalType> &input_table_types,
93960
93952
  vector<string> &input_table_names,
93961
93953
  vector<LogicalType> &return_types, vector<string> &names) {
@@ -94090,7 +94082,7 @@ struct DuckDBSettingsData : public FunctionOperatorData {
94090
94082
  };
94091
94083
 
94092
94084
  static unique_ptr<FunctionData> DuckDBSettingsBind(ClientContext &context, vector<Value> &inputs,
94093
- unordered_map<string, Value> &named_parameters,
94085
+ named_parameter_map_t &named_parameters,
94094
94086
  vector<LogicalType> &input_table_types,
94095
94087
  vector<string> &input_table_names, vector<LogicalType> &return_types,
94096
94088
  vector<string> &names) {
@@ -94199,7 +94191,7 @@ struct DuckDBTablesData : public FunctionOperatorData {
94199
94191
  };
94200
94192
 
94201
94193
  static unique_ptr<FunctionData> DuckDBTablesBind(ClientContext &context, vector<Value> &inputs,
94202
- unordered_map<string, Value> &named_parameters,
94194
+ named_parameter_map_t &named_parameters,
94203
94195
  vector<LogicalType> &input_table_types,
94204
94196
  vector<string> &input_table_names, vector<LogicalType> &return_types,
94205
94197
  vector<string> &names) {
@@ -94352,7 +94344,7 @@ struct DuckDBTypesData : public FunctionOperatorData {
94352
94344
  };
94353
94345
 
94354
94346
  static unique_ptr<FunctionData> DuckDBTypesBind(ClientContext &context, vector<Value> &inputs,
94355
- unordered_map<string, Value> &named_parameters,
94347
+ named_parameter_map_t &named_parameters,
94356
94348
  vector<LogicalType> &input_table_types,
94357
94349
  vector<string> &input_table_names, vector<LogicalType> &return_types,
94358
94350
  vector<string> &names) {
@@ -94491,7 +94483,7 @@ struct DuckDBViewsData : public FunctionOperatorData {
94491
94483
  };
94492
94484
 
94493
94485
  static unique_ptr<FunctionData> DuckDBViewsBind(ClientContext &context, vector<Value> &inputs,
94494
- unordered_map<string, Value> &named_parameters,
94486
+ named_parameter_map_t &named_parameters,
94495
94487
  vector<LogicalType> &input_table_types,
94496
94488
  vector<string> &input_table_names, vector<LogicalType> &return_types,
94497
94489
  vector<string> &names) {
@@ -94602,7 +94594,7 @@ struct PragmaCollateData : public FunctionOperatorData {
94602
94594
  };
94603
94595
 
94604
94596
  static unique_ptr<FunctionData> PragmaCollateBind(ClientContext &context, vector<Value> &inputs,
94605
- unordered_map<string, Value> &named_parameters,
94597
+ named_parameter_map_t &named_parameters,
94606
94598
  vector<LogicalType> &input_table_types,
94607
94599
  vector<string> &input_table_names, vector<LogicalType> &return_types,
94608
94600
  vector<string> &names) {
@@ -94662,7 +94654,7 @@ struct PragmaDatabaseListData : public FunctionOperatorData {
94662
94654
  };
94663
94655
 
94664
94656
  static unique_ptr<FunctionData> PragmaDatabaseListBind(ClientContext &context, vector<Value> &inputs,
94665
- unordered_map<string, Value> &named_parameters,
94657
+ named_parameter_map_t &named_parameters,
94666
94658
  vector<LogicalType> &input_table_types,
94667
94659
  vector<string> &input_table_names,
94668
94660
  vector<LogicalType> &return_types, vector<string> &names) {
@@ -94723,7 +94715,7 @@ struct PragmaDatabaseSizeData : public FunctionOperatorData {
94723
94715
  };
94724
94716
 
94725
94717
  static unique_ptr<FunctionData> PragmaDatabaseSizeBind(ClientContext &context, vector<Value> &inputs,
94726
- unordered_map<string, Value> &named_parameters,
94718
+ named_parameter_map_t &named_parameters,
94727
94719
  vector<LogicalType> &input_table_types,
94728
94720
  vector<string> &input_table_names,
94729
94721
  vector<LogicalType> &return_types, vector<string> &names) {
@@ -94827,7 +94819,7 @@ struct PragmaFunctionsData : public FunctionOperatorData {
94827
94819
  };
94828
94820
 
94829
94821
  static unique_ptr<FunctionData> PragmaFunctionsBind(ClientContext &context, vector<Value> &inputs,
94830
- unordered_map<string, Value> &named_parameters,
94822
+ named_parameter_map_t &named_parameters,
94831
94823
  vector<LogicalType> &input_table_types,
94832
94824
  vector<string> &input_table_names,
94833
94825
  vector<LogicalType> &return_types, vector<string> &names) {
@@ -94963,7 +94955,7 @@ struct PragmaStorageOperatorData : public FunctionOperatorData {
94963
94955
  };
94964
94956
 
94965
94957
  static unique_ptr<FunctionData> PragmaStorageInfoBind(ClientContext &context, vector<Value> &inputs,
94966
- unordered_map<string, Value> &named_parameters,
94958
+ named_parameter_map_t &named_parameters,
94967
94959
  vector<LogicalType> &input_table_types,
94968
94960
  vector<string> &input_table_names,
94969
94961
  vector<LogicalType> &return_types, vector<string> &names) {
@@ -95090,7 +95082,7 @@ struct PragmaTableOperatorData : public FunctionOperatorData {
95090
95082
  };
95091
95083
 
95092
95084
  static unique_ptr<FunctionData> PragmaTableInfoBind(ClientContext &context, vector<Value> &inputs,
95093
- unordered_map<string, Value> &named_parameters,
95085
+ named_parameter_map_t &named_parameters,
95094
95086
  vector<LogicalType> &input_table_types,
95095
95087
  vector<string> &input_table_names,
95096
95088
  vector<LogicalType> &return_types, vector<string> &names) {
@@ -95410,7 +95402,7 @@ static vector<TestType> GetTestTypes() {
95410
95402
  }
95411
95403
 
95412
95404
  static unique_ptr<FunctionData> TestAllTypesBind(ClientContext &context, vector<Value> &inputs,
95413
- unordered_map<string, Value> &named_parameters,
95405
+ named_parameter_map_t &named_parameters,
95414
95406
  vector<LogicalType> &input_table_types,
95415
95407
  vector<string> &input_table_names, vector<LogicalType> &return_types,
95416
95408
  vector<string> &names) {
@@ -95970,7 +95962,7 @@ struct UnnestOperatorData : public FunctionOperatorData {
95970
95962
  };
95971
95963
 
95972
95964
  static unique_ptr<FunctionData> UnnestBind(ClientContext &context, vector<Value> &inputs,
95973
- unordered_map<string, Value> &named_parameters,
95965
+ named_parameter_map_t &named_parameters,
95974
95966
  vector<LogicalType> &input_table_types, vector<string> &input_table_names,
95975
95967
  vector<LogicalType> &return_types, vector<string> &names) {
95976
95968
  return_types.push_back(ListType::GetChildType(inputs[0].type()));
@@ -96015,7 +96007,7 @@ struct PragmaVersionData : public FunctionOperatorData {
96015
96007
  };
96016
96008
 
96017
96009
  static unique_ptr<FunctionData> PragmaVersionBind(ClientContext &context, vector<Value> &inputs,
96018
- unordered_map<string, Value> &named_parameters,
96010
+ named_parameter_map_t &named_parameters,
96019
96011
  vector<LogicalType> &input_table_types,
96020
96012
  vector<string> &input_table_names, vector<LogicalType> &return_types,
96021
96013
  vector<string> &names) {
@@ -98757,7 +98749,7 @@ vector<unique_ptr<SQLStatement>> ClientContext::ParseStatements(const string &qu
98757
98749
  }
98758
98750
 
98759
98751
  vector<unique_ptr<SQLStatement>> ClientContext::ParseStatementsInternal(ClientContextLock &lock, const string &query) {
98760
- Parser parser;
98752
+ Parser parser(GetParserOptions());
98761
98753
  parser.ParseQuery(query);
98762
98754
 
98763
98755
  PragmaHandler handler(*this);
@@ -99494,6 +99486,12 @@ bool ClientContext::TryGetCurrentSetting(const std::string &key, Value &result)
99494
99486
  return true;
99495
99487
  }
99496
99488
 
99489
+ ParserOptions ClientContext::GetParserOptions() {
99490
+ ParserOptions options;
99491
+ options.preserve_identifier_case = ClientConfig::GetConfig(*this).preserve_identifier_case;
99492
+ return options;
99493
+ }
99494
+
99497
99495
  } // namespace duckdb
99498
99496
 
99499
99497
 
@@ -99687,6 +99685,15 @@ struct PerfectHashThresholdSetting {
99687
99685
  static Value GetSetting(ClientContext &context);
99688
99686
  };
99689
99687
 
99688
+ struct PreserveIdentifierCase {
99689
+ static constexpr const char *Name = "preserve_identifier_case";
99690
+ static constexpr const char *Description =
99691
+ "Whether or not to preserve the identifier case, instead of always lowercasing all non-quoted identifiers";
99692
+ static constexpr const LogicalTypeId InputType = LogicalTypeId::BOOLEAN;
99693
+ static void SetLocal(ClientContext &context, const Value &parameter);
99694
+ static Value GetSetting(ClientContext &context);
99695
+ };
99696
+
99690
99697
  struct ProfilerHistorySize {
99691
99698
  static constexpr const char *Name = "profiler_history_size";
99692
99699
  static constexpr const char *Description = "Sets the profiler history size";
@@ -99798,6 +99805,7 @@ static ConfigurationOption internal_options[] = {DUCKDB_GLOBAL(AccessModeSetting
99798
99805
  DUCKDB_GLOBAL_ALIAS("memory_limit", MaximumMemorySetting),
99799
99806
  DUCKDB_GLOBAL_ALIAS("null_order", DefaultNullOrderSetting),
99800
99807
  DUCKDB_LOCAL(PerfectHashThresholdSetting),
99808
+ DUCKDB_LOCAL(PreserveIdentifierCase),
99801
99809
  DUCKDB_LOCAL(ProfilerHistorySize),
99802
99810
  DUCKDB_LOCAL(ProfileOutputSetting),
99803
99811
  DUCKDB_LOCAL(ProfilingModeSetting),
@@ -99837,8 +99845,10 @@ ConfigurationOption *DBConfig::GetOptionByIndex(idx_t target_index) {
99837
99845
  }
99838
99846
 
99839
99847
  ConfigurationOption *DBConfig::GetOptionByName(const string &name) {
99848
+ auto lname = StringUtil::Lower(name);
99840
99849
  for (idx_t index = 0; internal_options[index].name; index++) {
99841
- if (internal_options[index].name == name) {
99850
+ D_ASSERT(StringUtil::Lower(internal_options[index].name) == string(internal_options[index].name));
99851
+ if (internal_options[index].name == lname) {
99842
99852
  return internal_options + index;
99843
99853
  }
99844
99854
  }
@@ -100041,15 +100051,14 @@ namespace duckdb {
100041
100051
  class TableFunctionRelation : public Relation {
100042
100052
  public:
100043
100053
  TableFunctionRelation(ClientContext &context, string name, vector<Value> parameters,
100044
- unordered_map<string, Value> named_parameters,
100045
- shared_ptr<Relation> input_relation_p = nullptr);
100054
+ named_parameter_map_t named_parameters, shared_ptr<Relation> input_relation_p = nullptr);
100046
100055
 
100047
100056
  TableFunctionRelation(ClientContext &context, string name, vector<Value> parameters,
100048
100057
  shared_ptr<Relation> input_relation_p = nullptr);
100049
100058
 
100050
100059
  string name;
100051
100060
  vector<Value> parameters;
100052
- unordered_map<string, Value> named_parameters;
100061
+ named_parameter_map_t named_parameters;
100053
100062
  vector<ColumnDefinition> columns;
100054
100063
  shared_ptr<Relation> input_relation;
100055
100064
 
@@ -100326,12 +100335,12 @@ shared_ptr<Relation> Connection::View(const string &schema_name, const string &t
100326
100335
 
100327
100336
  shared_ptr<Relation> Connection::TableFunction(const string &fname) {
100328
100337
  vector<Value> values;
100329
- unordered_map<string, Value> named_parameters;
100338
+ named_parameter_map_t named_parameters;
100330
100339
  return TableFunction(fname, values, named_parameters);
100331
100340
  }
100332
100341
 
100333
100342
  shared_ptr<Relation> Connection::TableFunction(const string &fname, const vector<Value> &values,
100334
- const unordered_map<string, Value> &named_parameters) {
100343
+ const named_parameter_map_t &named_parameters) {
100335
100344
  return make_shared<TableFunctionRelation>(*context, fname, values, named_parameters);
100336
100345
  }
100337
100346
 
@@ -100374,7 +100383,7 @@ shared_ptr<Relation> Connection::ReadCSV(const string &csv_file, const vector<st
100374
100383
  // parse columns
100375
100384
  vector<ColumnDefinition> column_list;
100376
100385
  for (auto &column : columns) {
100377
- auto col_list = Parser::ParseColumnList(column);
100386
+ auto col_list = Parser::ParseColumnList(column, context->GetParserOptions());
100378
100387
  if (col_list.size() != 1) {
100379
100388
  throw ParserException("Expected a single column definition");
100380
100389
  }
@@ -110918,7 +110927,7 @@ QueryRelation::QueryRelation(ClientContext &context, string query, string alias)
110918
110927
  }
110919
110928
 
110920
110929
  unique_ptr<SelectStatement> QueryRelation::GetSelectStatement() {
110921
- Parser parser;
110930
+ Parser parser(context.GetParserOptions());
110922
110931
  parser.ParseQuery(query);
110923
110932
  if (parser.statements.size() != 1) {
110924
110933
  throw ParserException("Expected a single SELECT statement");
@@ -111183,7 +111192,7 @@ string SubqueryRelation::ToString(idx_t depth) {
111183
111192
  namespace duckdb {
111184
111193
 
111185
111194
  TableFunctionRelation::TableFunctionRelation(ClientContext &context, string name_p, vector<Value> parameters_p,
111186
- unordered_map<string, Value> named_parameters,
111195
+ named_parameter_map_t named_parameters,
111187
111196
  shared_ptr<Relation> input_relation_p)
111188
111197
  : Relation(context, RelationType::TABLE_FUNCTION_RELATION), name(move(name_p)), parameters(move(parameters_p)),
111189
111198
  named_parameters(move(named_parameters)), input_relation(move(input_relation_p)) {
@@ -111300,6 +111309,7 @@ public:
111300
111309
 
111301
111310
 
111302
111311
 
111312
+
111303
111313
  namespace duckdb {
111304
111314
 
111305
111315
  TableRelation::TableRelation(ClientContext &context, unique_ptr<TableDescription> description)
@@ -111332,9 +111342,9 @@ string TableRelation::ToString(idx_t depth) {
111332
111342
  return RenderWhitespace(depth) + "Scan Table [" + description->table + "]";
111333
111343
  }
111334
111344
 
111335
- static unique_ptr<ParsedExpression> ParseCondition(const string &condition) {
111345
+ static unique_ptr<ParsedExpression> ParseCondition(ClientContext &context, const string &condition) {
111336
111346
  if (!condition.empty()) {
111337
- auto expression_list = Parser::ParseExpressionList(condition);
111347
+ auto expression_list = Parser::ParseExpressionList(condition, context.GetParserOptions());
111338
111348
  if (expression_list.size() != 1) {
111339
111349
  throw ParserException("Expected a single expression as filter condition");
111340
111350
  }
@@ -111347,15 +111357,15 @@ static unique_ptr<ParsedExpression> ParseCondition(const string &condition) {
111347
111357
  void TableRelation::Update(const string &update_list, const string &condition) {
111348
111358
  vector<string> update_columns;
111349
111359
  vector<unique_ptr<ParsedExpression>> expressions;
111350
- auto cond = ParseCondition(condition);
111351
- Parser::ParseUpdateList(update_list, update_columns, expressions);
111360
+ auto cond = ParseCondition(context, condition);
111361
+ Parser::ParseUpdateList(update_list, update_columns, expressions, context.GetParserOptions());
111352
111362
  auto update = make_shared<UpdateRelation>(context, move(cond), description->schema, description->table,
111353
111363
  move(update_columns), move(expressions));
111354
111364
  update->Execute();
111355
111365
  }
111356
111366
 
111357
111367
  void TableRelation::Delete(const string &condition) {
111358
- auto cond = ParseCondition(condition);
111368
+ auto cond = ParseCondition(context, condition);
111359
111369
  auto del = make_shared<DeleteRelation>(context, move(cond), description->schema, description->table);
111360
111370
  del->Execute();
111361
111371
  }
@@ -111468,7 +111478,7 @@ ValueRelation::ValueRelation(ClientContext &context, const vector<vector<Value>>
111468
111478
 
111469
111479
  ValueRelation::ValueRelation(ClientContext &context, const string &values_list, vector<string> names_p, string alias_p)
111470
111480
  : Relation(context, RelationType::VALUE_LIST_RELATION), names(move(names_p)), alias(move(alias_p)) {
111471
- this->expressions = Parser::ParseValuesList(values_list);
111481
+ this->expressions = Parser::ParseValuesList(values_list, context.GetParserOptions());
111472
111482
  context.TryBindRelation(*this, this->columns);
111473
111483
  }
111474
111484
 
@@ -111703,7 +111713,7 @@ shared_ptr<Relation> Relation::Project(const string &expression, const string &a
111703
111713
  }
111704
111714
 
111705
111715
  shared_ptr<Relation> Relation::Project(const string &select_list, const vector<string> &aliases) {
111706
- auto expressions = Parser::ParseExpressionList(select_list);
111716
+ auto expressions = Parser::ParseExpressionList(select_list, context.GetParserOptions());
111707
111717
  return make_shared<ProjectionRelation>(shared_from_this(), move(expressions), aliases);
111708
111718
  }
111709
111719
 
@@ -111712,13 +111722,14 @@ shared_ptr<Relation> Relation::Project(const vector<string> &expressions) {
111712
111722
  return Project(expressions, aliases);
111713
111723
  }
111714
111724
 
111715
- static vector<unique_ptr<ParsedExpression>> StringListToExpressionList(const vector<string> &expressions) {
111725
+ static vector<unique_ptr<ParsedExpression>> StringListToExpressionList(ClientContext &context,
111726
+ const vector<string> &expressions) {
111716
111727
  if (expressions.empty()) {
111717
111728
  throw ParserException("Zero expressions provided");
111718
111729
  }
111719
111730
  vector<unique_ptr<ParsedExpression>> result_list;
111720
111731
  for (auto &expr : expressions) {
111721
- auto expression_list = Parser::ParseExpressionList(expr);
111732
+ auto expression_list = Parser::ParseExpressionList(expr, context.GetParserOptions());
111722
111733
  if (expression_list.size() != 1) {
111723
111734
  throw ParserException("Expected a single expression in the expression list");
111724
111735
  }
@@ -111728,12 +111739,12 @@ static vector<unique_ptr<ParsedExpression>> StringListToExpressionList(const vec
111728
111739
  }
111729
111740
 
111730
111741
  shared_ptr<Relation> Relation::Project(const vector<string> &expressions, const vector<string> &aliases) {
111731
- auto result_list = StringListToExpressionList(expressions);
111742
+ auto result_list = StringListToExpressionList(context, expressions);
111732
111743
  return make_shared<ProjectionRelation>(shared_from_this(), move(result_list), aliases);
111733
111744
  }
111734
111745
 
111735
111746
  shared_ptr<Relation> Relation::Filter(const string &expression) {
111736
- auto expression_list = Parser::ParseExpressionList(expression);
111747
+ auto expression_list = Parser::ParseExpressionList(expression, context.GetParserOptions());
111737
111748
  if (expression_list.size() != 1) {
111738
111749
  throw ParserException("Expected a single expression as filter condition");
111739
111750
  }
@@ -111742,7 +111753,7 @@ shared_ptr<Relation> Relation::Filter(const string &expression) {
111742
111753
 
111743
111754
  shared_ptr<Relation> Relation::Filter(const vector<string> &expressions) {
111744
111755
  // if there are multiple expressions, we AND them together
111745
- auto expression_list = StringListToExpressionList(expressions);
111756
+ auto expression_list = StringListToExpressionList(context, expressions);
111746
111757
  D_ASSERT(!expression_list.empty());
111747
111758
 
111748
111759
  auto expr = move(expression_list[0]);
@@ -111758,7 +111769,7 @@ shared_ptr<Relation> Relation::Limit(int64_t limit, int64_t offset) {
111758
111769
  }
111759
111770
 
111760
111771
  shared_ptr<Relation> Relation::Order(const string &expression) {
111761
- auto order_list = Parser::ParseOrderList(expression);
111772
+ auto order_list = Parser::ParseOrderList(expression, context.GetParserOptions());
111762
111773
  return make_shared<OrderRelation>(shared_from_this(), move(order_list));
111763
111774
  }
111764
111775
 
@@ -111768,7 +111779,7 @@ shared_ptr<Relation> Relation::Order(const vector<string> &expressions) {
111768
111779
  }
111769
111780
  vector<OrderByNode> order_list;
111770
111781
  for (auto &expression : expressions) {
111771
- auto inner_list = Parser::ParseOrderList(expression);
111782
+ auto inner_list = Parser::ParseOrderList(expression, context.GetParserOptions());
111772
111783
  if (inner_list.size() != 1) {
111773
111784
  throw ParserException("Expected a single ORDER BY expression in the expression list");
111774
111785
  }
@@ -111778,7 +111789,7 @@ shared_ptr<Relation> Relation::Order(const vector<string> &expressions) {
111778
111789
  }
111779
111790
 
111780
111791
  shared_ptr<Relation> Relation::Join(const shared_ptr<Relation> &other, const string &condition, JoinType type) {
111781
- auto expression_list = Parser::ParseExpressionList(condition);
111792
+ auto expression_list = Parser::ParseExpressionList(condition, context.GetParserOptions());
111782
111793
  D_ASSERT(!expression_list.empty());
111783
111794
 
111784
111795
  if (expression_list.size() > 1 || expression_list[0]->type == ExpressionType::COLUMN_REF) {
@@ -111822,24 +111833,24 @@ shared_ptr<Relation> Relation::Alias(const string &alias) {
111822
111833
  }
111823
111834
 
111824
111835
  shared_ptr<Relation> Relation::Aggregate(const string &aggregate_list) {
111825
- auto expression_list = Parser::ParseExpressionList(aggregate_list);
111836
+ auto expression_list = Parser::ParseExpressionList(aggregate_list, context.GetParserOptions());
111826
111837
  return make_shared<AggregateRelation>(shared_from_this(), move(expression_list));
111827
111838
  }
111828
111839
 
111829
111840
  shared_ptr<Relation> Relation::Aggregate(const string &aggregate_list, const string &group_list) {
111830
- auto expression_list = Parser::ParseExpressionList(aggregate_list);
111831
- auto groups = Parser::ParseExpressionList(group_list);
111841
+ auto expression_list = Parser::ParseExpressionList(aggregate_list, context.GetParserOptions());
111842
+ auto groups = Parser::ParseExpressionList(group_list, context.GetParserOptions());
111832
111843
  return make_shared<AggregateRelation>(shared_from_this(), move(expression_list), move(groups));
111833
111844
  }
111834
111845
 
111835
111846
  shared_ptr<Relation> Relation::Aggregate(const vector<string> &aggregates) {
111836
- auto aggregate_list = StringListToExpressionList(aggregates);
111847
+ auto aggregate_list = StringListToExpressionList(context, aggregates);
111837
111848
  return make_shared<AggregateRelation>(shared_from_this(), move(aggregate_list));
111838
111849
  }
111839
111850
 
111840
111851
  shared_ptr<Relation> Relation::Aggregate(const vector<string> &aggregates, const vector<string> &groups) {
111841
- auto aggregate_list = StringListToExpressionList(aggregates);
111842
- auto group_list = StringListToExpressionList(groups);
111852
+ auto aggregate_list = StringListToExpressionList(context, aggregates);
111853
+ auto group_list = StringListToExpressionList(context, groups);
111843
111854
  return make_shared<AggregateRelation>(shared_from_this(), move(aggregate_list), move(group_list));
111844
111855
  }
111845
111856
 
@@ -111933,7 +111944,7 @@ void Relation::Delete(const string &condition) {
111933
111944
  }
111934
111945
 
111935
111946
  shared_ptr<Relation> Relation::TableFunction(const std::string &fname, const vector<Value> &values,
111936
- const unordered_map<string, Value> &named_parameters) {
111947
+ const named_parameter_map_t &named_parameters) {
111937
111948
  return make_shared<TableFunctionRelation>(context, fname, values, named_parameters, shared_from_this());
111938
111949
  }
111939
111950
 
@@ -111989,6 +112000,7 @@ string Relation::RenderWhitespace(idx_t depth) {
111989
112000
 
111990
112001
 
111991
112002
 
112003
+
111992
112004
  namespace duckdb {
111993
112005
 
111994
112006
  //===--------------------------------------------------------------------===//
@@ -112386,6 +112398,17 @@ Value PerfectHashThresholdSetting::GetSetting(ClientContext &context) {
112386
112398
  return Value::BIGINT(ClientConfig::GetConfig(context).perfect_ht_threshold);
112387
112399
  }
112388
112400
 
112401
+ //===--------------------------------------------------------------------===//
112402
+ // PreserveIdentifierCase
112403
+ //===--------------------------------------------------------------------===//
112404
+ void PreserveIdentifierCase::SetLocal(ClientContext &context, const Value &input) {
112405
+ ClientConfig::GetConfig(context).preserve_identifier_case = input.GetValue<bool>();
112406
+ }
112407
+
112408
+ Value PreserveIdentifierCase::GetSetting(ClientContext &context) {
112409
+ return Value::BOOLEAN(ClientConfig::GetConfig(context).preserve_identifier_case);
112410
+ }
112411
+
112389
112412
  //===--------------------------------------------------------------------===//
112390
112413
  // Profiler History Size
112391
112414
  //===--------------------------------------------------------------------===//
@@ -127785,6 +127808,7 @@ unique_ptr<ParsedExpression> CollateExpression::Deserialize(ExpressionType type,
127785
127808
 
127786
127809
 
127787
127810
 
127811
+
127788
127812
  namespace duckdb {
127789
127813
 
127790
127814
  ColumnRefExpression::ColumnRefExpression(string column_name, string table_name)
@@ -127834,13 +127858,24 @@ string ColumnRefExpression::ToString() const {
127834
127858
  }
127835
127859
 
127836
127860
  bool ColumnRefExpression::Equals(const ColumnRefExpression *a, const ColumnRefExpression *b) {
127837
- return a->column_names == b->column_names;
127861
+ if (a->column_names.size() != b->column_names.size()) {
127862
+ return false;
127863
+ }
127864
+ for (idx_t i = 0; i < a->column_names.size(); i++) {
127865
+ auto lcase_a = StringUtil::Lower(a->column_names[i]);
127866
+ auto lcase_b = StringUtil::Lower(b->column_names[i]);
127867
+ if (lcase_a != lcase_b) {
127868
+ return false;
127869
+ }
127870
+ }
127871
+ return true;
127838
127872
  }
127839
127873
 
127840
127874
  hash_t ColumnRefExpression::Hash() const {
127841
127875
  hash_t result = ParsedExpression::Hash();
127842
127876
  for (auto &column_name : column_names) {
127843
- result = CombineHash(result, duckdb::Hash<const char *>(column_name.c_str()));
127877
+ auto lcase = StringUtil::Lower(column_name);
127878
+ result = CombineHash(result, duckdb::Hash<const char *>(lcase.c_str()));
127844
127879
  }
127845
127880
  return result;
127846
127881
  }
@@ -135063,7 +135098,10 @@ public:
135063
135098
 
135064
135099
  static bool IsKeyword(const std::string &text);
135065
135100
  static std::vector<duckdb_libpgquery::PGKeyword> KeywordList();
135101
+
135102
+ static void SetPreserveIdentifierCase(bool downcase);
135066
135103
  };
135104
+
135067
135105
  }
135068
135106
 
135069
135107
 
@@ -135124,12 +135162,13 @@ PGTypeName *SystemTypeName(const char *name);
135124
135162
 
135125
135163
  namespace duckdb {
135126
135164
 
135127
- Parser::Parser() {
135165
+ Parser::Parser(ParserOptions options_p) : options(options_p) {
135128
135166
  }
135129
135167
 
135130
135168
  void Parser::ParseQuery(const string &query) {
135131
135169
  Transformer transformer;
135132
135170
  {
135171
+ PostgresParser::SetPreserveIdentifierCase(options.preserve_identifier_case);
135133
135172
  PostgresParser parser;
135134
135173
  parser.Parse(query);
135135
135174
 
@@ -135225,11 +135264,11 @@ vector<ParserKeyword> Parser::KeywordList() {
135225
135264
  return result;
135226
135265
  }
135227
135266
 
135228
- vector<unique_ptr<ParsedExpression>> Parser::ParseExpressionList(const string &select_list) {
135267
+ vector<unique_ptr<ParsedExpression>> Parser::ParseExpressionList(const string &select_list, ParserOptions options) {
135229
135268
  // construct a mock query prefixed with SELECT
135230
135269
  string mock_query = "SELECT " + select_list;
135231
135270
  // parse the query
135232
- Parser parser;
135271
+ Parser parser(options);
135233
135272
  parser.ParseQuery(mock_query);
135234
135273
  // check the statements
135235
135274
  if (parser.statements.size() != 1 || parser.statements[0]->type != StatementType::SELECT_STATEMENT) {
@@ -135243,11 +135282,11 @@ vector<unique_ptr<ParsedExpression>> Parser::ParseExpressionList(const string &s
135243
135282
  return move(select_node.select_list);
135244
135283
  }
135245
135284
 
135246
- vector<OrderByNode> Parser::ParseOrderList(const string &select_list) {
135285
+ vector<OrderByNode> Parser::ParseOrderList(const string &select_list, ParserOptions options) {
135247
135286
  // construct a mock query
135248
135287
  string mock_query = "SELECT * FROM tbl ORDER BY " + select_list;
135249
135288
  // parse the query
135250
- Parser parser;
135289
+ Parser parser(options);
135251
135290
  parser.ParseQuery(mock_query);
135252
135291
  // check the statements
135253
135292
  if (parser.statements.size() != 1 || parser.statements[0]->type != StatementType::SELECT_STATEMENT) {
@@ -135267,11 +135306,11 @@ vector<OrderByNode> Parser::ParseOrderList(const string &select_list) {
135267
135306
  }
135268
135307
 
135269
135308
  void Parser::ParseUpdateList(const string &update_list, vector<string> &update_columns,
135270
- vector<unique_ptr<ParsedExpression>> &expressions) {
135309
+ vector<unique_ptr<ParsedExpression>> &expressions, ParserOptions options) {
135271
135310
  // construct a mock query
135272
135311
  string mock_query = "UPDATE tbl SET " + update_list;
135273
135312
  // parse the query
135274
- Parser parser;
135313
+ Parser parser(options);
135275
135314
  parser.ParseQuery(mock_query);
135276
135315
  // check the statements
135277
135316
  if (parser.statements.size() != 1 || parser.statements[0]->type != StatementType::UPDATE_STATEMENT) {
@@ -135282,11 +135321,11 @@ void Parser::ParseUpdateList(const string &update_list, vector<string> &update_c
135282
135321
  expressions = move(update.expressions);
135283
135322
  }
135284
135323
 
135285
- vector<vector<unique_ptr<ParsedExpression>>> Parser::ParseValuesList(const string &value_list) {
135324
+ vector<vector<unique_ptr<ParsedExpression>>> Parser::ParseValuesList(const string &value_list, ParserOptions options) {
135286
135325
  // construct a mock query
135287
135326
  string mock_query = "VALUES " + value_list;
135288
135327
  // parse the query
135289
- Parser parser;
135328
+ Parser parser(options);
135290
135329
  parser.ParseQuery(mock_query);
135291
135330
  // check the statements
135292
135331
  if (parser.statements.size() != 1 || parser.statements[0]->type != StatementType::SELECT_STATEMENT) {
@@ -135304,9 +135343,9 @@ vector<vector<unique_ptr<ParsedExpression>>> Parser::ParseValuesList(const strin
135304
135343
  return move(values_list.values);
135305
135344
  }
135306
135345
 
135307
- vector<ColumnDefinition> Parser::ParseColumnList(const string &column_list) {
135346
+ vector<ColumnDefinition> Parser::ParseColumnList(const string &column_list, ParserOptions options) {
135308
135347
  string mock_query = "CREATE TABLE blabla (" + column_list + ")";
135309
- Parser parser;
135348
+ Parser parser(options);
135310
135349
  parser.ParseQuery(mock_query);
135311
135350
  if (parser.statements.size() != 1 || parser.statements[0]->type != StatementType::CREATE_STATEMENT) {
135312
135351
  throw ParserException("Expected a single CREATE statement");
@@ -142142,7 +142181,7 @@ class BoundSelectNode;
142142
142181
 
142143
142182
  struct BoundGroupInformation {
142144
142183
  expression_map_t<idx_t> map;
142145
- unordered_map<string, idx_t> alias_map;
142184
+ case_insensitive_map_t<idx_t> alias_map;
142146
142185
  };
142147
142186
 
142148
142187
  //! The SELECT binder is responsible for binding an expression within the SELECT clause of a SQL statement
@@ -142928,6 +142967,7 @@ unique_ptr<Expression> ExpressionBinder::PushCollation(ClientContext &context, u
142928
142967
  } else {
142929
142968
  collation = collation_p;
142930
142969
  }
142970
+ collation = StringUtil::Lower(collation);
142931
142971
  // bind the collation
142932
142972
  if (collation.empty() || collation == "binary" || collation == "c" || collation == "posix") {
142933
142973
  // binary collation: just skip
@@ -144006,14 +144046,14 @@ class ColumnRefExpression;
144006
144046
  //! A helper binder for WhereBinder and HavingBinder which support alias as a columnref.
144007
144047
  class ColumnAliasBinder {
144008
144048
  public:
144009
- ColumnAliasBinder(BoundSelectNode &node, const unordered_map<string, idx_t> &alias_map);
144049
+ ColumnAliasBinder(BoundSelectNode &node, const case_insensitive_map_t<idx_t> &alias_map);
144010
144050
 
144011
144051
  BindResult BindAlias(ExpressionBinder &enclosing_binder, ColumnRefExpression &expr, idx_t depth,
144012
144052
  bool root_expression);
144013
144053
 
144014
144054
  private:
144015
144055
  BoundSelectNode &node;
144016
- const unordered_map<string, idx_t> &alias_map;
144056
+ const case_insensitive_map_t<idx_t> &alias_map;
144017
144057
  bool in_alias;
144018
144058
  };
144019
144059
 
@@ -144062,7 +144102,6 @@ protected:
144062
144102
 
144063
144103
 
144064
144104
 
144065
-
144066
144105
  namespace duckdb {
144067
144106
  class ConstantExpression;
144068
144107
  class ColumnRefExpression;
@@ -144071,7 +144110,7 @@ class ColumnRefExpression;
144071
144110
  class GroupBinder : public ExpressionBinder {
144072
144111
  public:
144073
144112
  GroupBinder(Binder &binder, ClientContext &context, SelectNode &node, idx_t group_index,
144074
- unordered_map<string, idx_t> &alias_map, unordered_map<string, idx_t> &group_alias_map);
144113
+ case_insensitive_map_t<idx_t> &alias_map, case_insensitive_map_t<idx_t> &group_alias_map);
144075
144114
 
144076
144115
  //! The unbound root expression
144077
144116
  unique_ptr<ParsedExpression> unbound_expression;
@@ -144088,8 +144127,8 @@ protected:
144088
144127
  BindResult BindConstant(ConstantExpression &expr);
144089
144128
 
144090
144129
  SelectNode &node;
144091
- unordered_map<string, idx_t> &alias_map;
144092
- unordered_map<string, idx_t> &group_alias_map;
144130
+ case_insensitive_map_t<idx_t> &alias_map;
144131
+ case_insensitive_map_t<idx_t> &group_alias_map;
144093
144132
  unordered_set<idx_t> used_aliases;
144094
144133
 
144095
144134
  idx_t group_index;
@@ -144116,7 +144155,7 @@ namespace duckdb {
144116
144155
  class HavingBinder : public SelectBinder {
144117
144156
  public:
144118
144157
  HavingBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info,
144119
- unordered_map<string, idx_t> &alias_map);
144158
+ case_insensitive_map_t<idx_t> &alias_map);
144120
144159
 
144121
144160
  protected:
144122
144161
  BindResult BindExpression(unique_ptr<ParsedExpression> *expr_ptr, idx_t depth,
@@ -144149,7 +144188,7 @@ namespace duckdb {
144149
144188
  class QualifyBinder : public SelectBinder {
144150
144189
  public:
144151
144190
  QualifyBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info,
144152
- unordered_map<string, idx_t> &alias_map);
144191
+ case_insensitive_map_t<idx_t> &alias_map);
144153
144192
 
144154
144193
  protected:
144155
144194
  BindResult BindExpression(unique_ptr<ParsedExpression> *expr_ptr, idx_t depth,
@@ -144186,10 +144225,10 @@ class SelectNode;
144186
144225
  //! The ORDER binder is responsible for binding an expression within the ORDER BY clause of a SQL statement
144187
144226
  class OrderBinder {
144188
144227
  public:
144189
- OrderBinder(vector<Binder *> binders, idx_t projection_index, unordered_map<string, idx_t> &alias_map,
144228
+ OrderBinder(vector<Binder *> binders, idx_t projection_index, case_insensitive_map_t<idx_t> &alias_map,
144190
144229
  expression_map_t<idx_t> &projection_map, idx_t max_count);
144191
144230
  OrderBinder(vector<Binder *> binders, idx_t projection_index, SelectNode &node,
144192
- unordered_map<string, idx_t> &alias_map, expression_map_t<idx_t> &projection_map);
144231
+ case_insensitive_map_t<idx_t> &alias_map, expression_map_t<idx_t> &projection_map);
144193
144232
 
144194
144233
  public:
144195
144234
  unique_ptr<Expression> Bind(unique_ptr<ParsedExpression> expr);
@@ -144206,7 +144245,7 @@ private:
144206
144245
  idx_t projection_index;
144207
144246
  idx_t max_count;
144208
144247
  vector<unique_ptr<ParsedExpression>> *extra_list;
144209
- unordered_map<string, idx_t> &alias_map;
144248
+ case_insensitive_map_t<idx_t> &alias_map;
144210
144249
  expression_map_t<idx_t> &projection_map;
144211
144250
  };
144212
144251
 
@@ -144450,7 +144489,7 @@ unique_ptr<BoundQueryNode> Binder::BindNode(SelectNode &statement) {
144450
144489
  statement.select_list = move(new_select_list);
144451
144490
 
144452
144491
  // create a mapping of (alias -> index) and a mapping of (Expression -> index) for the SELECT list
144453
- unordered_map<string, idx_t> alias_map;
144492
+ case_insensitive_map_t<idx_t> alias_map;
144454
144493
  expression_map_t<idx_t> projection_map;
144455
144494
  for (idx_t i = 0; i < statement.select_list.size(); i++) {
144456
144495
  auto &expr = statement.select_list[i];
@@ -144644,7 +144683,7 @@ public:
144644
144683
 
144645
144684
  namespace duckdb {
144646
144685
 
144647
- static void GatherAliases(BoundQueryNode &node, unordered_map<string, idx_t> &aliases,
144686
+ static void GatherAliases(BoundQueryNode &node, case_insensitive_map_t<idx_t> &aliases,
144648
144687
  expression_map_t<idx_t> &expressions) {
144649
144688
  if (node.type == QueryNodeType::SET_OPERATION_NODE) {
144650
144689
  // setop, recurse
@@ -144714,7 +144753,7 @@ unique_ptr<BoundQueryNode> Binder::BindNode(SetOperationNode &statement) {
144714
144753
 
144715
144754
  // we recursively visit the children of this node to extract aliases and expressions that can be referenced in
144716
144755
  // the ORDER BY
144717
- unordered_map<string, idx_t> alias_map;
144756
+ case_insensitive_map_t<idx_t> alias_map;
144718
144757
  expression_map_t<idx_t> expression_map;
144719
144758
  GatherAliases(*result, alias_map, expression_map);
144720
144759
 
@@ -146594,7 +146633,7 @@ BoundStatement Binder::Bind(InsertStatement &stmt) {
146594
146633
  // insertion statement specifies column list
146595
146634
 
146596
146635
  // create a mapping of (list index) -> (column index)
146597
- unordered_map<string, idx_t> column_name_map;
146636
+ case_insensitive_map_t<idx_t> column_name_map;
146598
146637
  for (idx_t i = 0; i < stmt.columns.size(); i++) {
146599
146638
  column_name_map[stmt.columns[i]] = i;
146600
146639
  auto entry = table->name_map.find(stmt.columns[i]);
@@ -147905,7 +147944,7 @@ unique_ptr<BoundTableRef> Binder::Bind(JoinRef &ref) {
147905
147944
 
147906
147945
  namespace duckdb {
147907
147946
 
147908
- void Binder::BindNamedParameters(unordered_map<string, LogicalType> &types, unordered_map<string, Value> &values,
147947
+ void Binder::BindNamedParameters(named_parameter_type_map_t &types, named_parameter_map_t &values,
147909
147948
  QueryErrorContext &error_context, string &func_name) {
147910
147949
  for (auto &kv : values) {
147911
147950
  auto entry = types.find(kv.first);
@@ -147982,7 +148021,7 @@ unique_ptr<BoundTableRef> Binder::Bind(SubqueryRef &ref, CommonTableExpressionIn
147982
148021
  namespace duckdb {
147983
148022
 
147984
148023
  bool Binder::BindFunctionParameters(vector<unique_ptr<ParsedExpression>> &expressions, vector<LogicalType> &arguments,
147985
- vector<Value> &parameters, unordered_map<string, Value> &named_parameters,
148024
+ vector<Value> &parameters, named_parameter_map_t &named_parameters,
147986
148025
  unique_ptr<BoundSubqueryRef> &subquery, string &error) {
147987
148026
  bool seen_subquery = false;
147988
148027
  for (auto &child : expressions) {
@@ -148046,7 +148085,7 @@ unique_ptr<BoundTableRef> Binder::Bind(TableFunctionRef &ref) {
148046
148085
  // evaluate the input parameters to the function
148047
148086
  vector<LogicalType> arguments;
148048
148087
  vector<Value> parameters;
148049
- unordered_map<string, Value> named_parameters;
148088
+ named_parameter_map_t named_parameters;
148050
148089
  unique_ptr<BoundSubqueryRef> subquery;
148051
148090
  string error;
148052
148091
  if (!BindFunctionParameters(fexpr->children, arguments, parameters, named_parameters, subquery, error)) {
@@ -150013,7 +150052,7 @@ BindResult CheckBinder::BindCheckColumn(ColumnRefExpression &colref) {
150013
150052
 
150014
150053
  namespace duckdb {
150015
150054
 
150016
- ColumnAliasBinder::ColumnAliasBinder(BoundSelectNode &node, const unordered_map<string, idx_t> &alias_map)
150055
+ ColumnAliasBinder::ColumnAliasBinder(BoundSelectNode &node, const case_insensitive_map_t<idx_t> &alias_map)
150017
150056
  : node(node), alias_map(alias_map), in_alias(false) {
150018
150057
  }
150019
150058
 
@@ -150077,7 +150116,7 @@ string ConstantBinder::UnsupportedAggregateMessage() {
150077
150116
  namespace duckdb {
150078
150117
 
150079
150118
  GroupBinder::GroupBinder(Binder &binder, ClientContext &context, SelectNode &node, idx_t group_index,
150080
- unordered_map<string, idx_t> &alias_map, unordered_map<string, idx_t> &group_alias_map)
150119
+ case_insensitive_map_t<idx_t> &alias_map, case_insensitive_map_t<idx_t> &group_alias_map)
150081
150120
  : ExpressionBinder(binder, context), node(node), alias_map(alias_map), group_alias_map(group_alias_map),
150082
150121
  group_index(group_index) {
150083
150122
  }
@@ -150185,7 +150224,7 @@ BindResult GroupBinder::BindColumnRef(ColumnRefExpression &colref) {
150185
150224
  namespace duckdb {
150186
150225
 
150187
150226
  HavingBinder::HavingBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info,
150188
- unordered_map<string, idx_t> &alias_map)
150227
+ case_insensitive_map_t<idx_t> &alias_map)
150189
150228
  : SelectBinder(binder, context, node, info), column_alias_binder(node, alias_map) {
150190
150229
  target_type = LogicalType(LogicalTypeId::BOOLEAN);
150191
150230
  }
@@ -150280,13 +150319,13 @@ string InsertBinder::UnsupportedAggregateMessage() {
150280
150319
 
150281
150320
  namespace duckdb {
150282
150321
 
150283
- OrderBinder::OrderBinder(vector<Binder *> binders, idx_t projection_index, unordered_map<string, idx_t> &alias_map,
150322
+ OrderBinder::OrderBinder(vector<Binder *> binders, idx_t projection_index, case_insensitive_map_t<idx_t> &alias_map,
150284
150323
  expression_map_t<idx_t> &projection_map, idx_t max_count)
150285
150324
  : binders(move(binders)), projection_index(projection_index), max_count(max_count), extra_list(nullptr),
150286
150325
  alias_map(alias_map), projection_map(projection_map) {
150287
150326
  }
150288
150327
  OrderBinder::OrderBinder(vector<Binder *> binders, idx_t projection_index, SelectNode &node,
150289
- unordered_map<string, idx_t> &alias_map, expression_map_t<idx_t> &projection_map)
150328
+ case_insensitive_map_t<idx_t> &alias_map, expression_map_t<idx_t> &projection_map)
150290
150329
  : binders(move(binders)), projection_index(projection_index), alias_map(alias_map), projection_map(projection_map) {
150291
150330
  this->max_count = node.select_list.size();
150292
150331
  this->extra_list = &node.select_list;
@@ -150384,7 +150423,7 @@ unique_ptr<Expression> OrderBinder::Bind(unique_ptr<ParsedExpression> expr) {
150384
150423
  namespace duckdb {
150385
150424
 
150386
150425
  QualifyBinder::QualifyBinder(Binder &binder, ClientContext &context, BoundSelectNode &node, BoundGroupInformation &info,
150387
- unordered_map<string, idx_t> &alias_map)
150426
+ case_insensitive_map_t<idx_t> &alias_map)
150388
150427
  : SelectBinder(binder, context, node, info), column_alias_binder(node, alias_map) {
150389
150428
  target_type = LogicalType(LogicalTypeId::BOOLEAN);
150390
150429
  }
@@ -152421,7 +152460,7 @@ void PragmaHandler::HandlePragmaStatementsInternal(vector<unique_ptr<SQLStatemen
152421
152460
  if (!new_query.empty()) {
152422
152461
  // this PRAGMA statement gets replaced by a new query string
152423
152462
  // push the new query string through the parser again and add it to the transformer
152424
- Parser parser;
152463
+ Parser parser(context.GetParserOptions());
152425
152464
  parser.ParseQuery(new_query);
152426
152465
  // insert the new statements and remove the old statement
152427
152466
  // FIXME: off by one here maybe?
@@ -222210,6 +222249,45 @@ PGNode *newNode(size_t size, PGNodeTag type) {
222210
222249
 
222211
222250
 
222212
222251
 
222252
+ // LICENSE_CHANGE_BEGIN
222253
+ // The following code up to LICENSE_CHANGE_END is subject to THIRD PARTY LICENSE #9
222254
+ // See the end of this file for a list
222255
+
222256
+ /*-------------------------------------------------------------------------
222257
+ *
222258
+ * scansup.h
222259
+ * scanner support routines. used by both the bootstrap lexer
222260
+ * as well as the normal lexer
222261
+ *
222262
+ * Portions Copyright (c) 1996-2017, PostgreSQL Global Development PGGroup
222263
+ * Portions Copyright (c) 1994, Regents of the University of California
222264
+ *
222265
+ * src/include/parser/scansup.h
222266
+ *
222267
+ *-------------------------------------------------------------------------
222268
+ */
222269
+
222270
+
222271
+
222272
+ namespace duckdb_libpgquery {
222273
+
222274
+ char *scanstr(const char *s);
222275
+
222276
+ char *downcase_truncate_identifier(const char *ident, int len, bool warn);
222277
+
222278
+ char *downcase_identifier(const char *ident, int len, bool warn, bool truncate);
222279
+
222280
+ bool scanner_isspace(char ch);
222281
+
222282
+ void set_preserve_identifier_case(bool downcase);
222283
+ bool get_preserve_identifier_case();
222284
+
222285
+ }
222286
+
222287
+ // LICENSE_CHANGE_END
222288
+
222289
+
222290
+
222213
222291
  // LICENSE_CHANGE_BEGIN
222214
222292
  // The following code up to LICENSE_CHANGE_END is subject to THIRD PARTY LICENSE #9
222215
222293
  // See the end of this file for a list
@@ -222290,6 +222368,10 @@ vector<duckdb_libpgquery::PGKeyword> PostgresParser::KeywordList() {
222290
222368
  return duckdb_libpgquery::keyword_list();
222291
222369
  }
222292
222370
 
222371
+ void PostgresParser::SetPreserveIdentifierCase(bool preserve) {
222372
+ duckdb_libpgquery::set_preserve_identifier_case(preserve);
222373
+ }
222374
+
222293
222375
  }
222294
222376
 
222295
222377
 
@@ -251634,41 +251716,6 @@ int base_yylex(YYSTYPE *lvalp, YYLTYPE *llocp, core_yyscan_t yyscanner) {
251634
251716
  /* only needed for GUC variables */
251635
251717
 
251636
251718
 
251637
- // LICENSE_CHANGE_BEGIN
251638
- // The following code up to LICENSE_CHANGE_END is subject to THIRD PARTY LICENSE #9
251639
- // See the end of this file for a list
251640
-
251641
- /*-------------------------------------------------------------------------
251642
- *
251643
- * scansup.h
251644
- * scanner support routines. used by both the bootstrap lexer
251645
- * as well as the normal lexer
251646
- *
251647
- * Portions Copyright (c) 1996-2017, PostgreSQL Global Development PGGroup
251648
- * Portions Copyright (c) 1994, Regents of the University of California
251649
- *
251650
- * src/include/parser/scansup.h
251651
- *
251652
- *-------------------------------------------------------------------------
251653
- */
251654
-
251655
-
251656
-
251657
- namespace duckdb_libpgquery {
251658
-
251659
- char *scanstr(const char *s);
251660
-
251661
- char *downcase_truncate_identifier(const char *ident, int len, bool warn);
251662
-
251663
- char *downcase_identifier(const char *ident, int len, bool warn, bool truncate);
251664
-
251665
- bool scanner_isspace(char ch);
251666
-
251667
- }
251668
-
251669
- // LICENSE_CHANGE_END
251670
-
251671
-
251672
251719
 
251673
251720
  // LICENSE_CHANGE_BEGIN
251674
251721
  // The following code up to LICENSE_CHANGE_END is subject to THIRD PARTY LICENSE #9
@@ -255828,6 +255875,16 @@ char *downcase_truncate_identifier(const char *ident, int len, bool warn) {
255828
255875
  return downcase_identifier(ident, len, warn, true);
255829
255876
  }
255830
255877
 
255878
+ static __thread bool pg_preserve_identifier_case = false;
255879
+
255880
+ void set_preserve_identifier_case(bool preserve) {
255881
+ pg_preserve_identifier_case = preserve;
255882
+ }
255883
+
255884
+ bool get_preserve_identifier_case() {
255885
+ return pg_preserve_identifier_case;
255886
+ }
255887
+
255831
255888
  /*
255832
255889
  * a workhorse for downcase_truncate_identifier
255833
255890
  */
@@ -255851,10 +255908,12 @@ char *downcase_identifier(const char *ident, int len, bool warn, bool truncate)
255851
255908
  for (i = 0; i < len; i++) {
255852
255909
  unsigned char ch = (unsigned char)ident[i];
255853
255910
 
255854
- if (ch >= 'A' && ch <= 'Z')
255855
- ch += 'a' - 'A';
255856
- else if (enc_is_single_byte && IS_HIGHBIT_SET(ch) && isupper(ch))
255857
- ch = tolower(ch);
255911
+ if (!get_preserve_identifier_case()) {
255912
+ if (ch >= 'A' && ch <= 'Z')
255913
+ ch += 'a' - 'A';
255914
+ else if (enc_is_single_byte && IS_HIGHBIT_SET(ch) && isupper(ch))
255915
+ ch = tolower(ch);
255916
+ }
255858
255917
  result[i] = (char)ch;
255859
255918
  }
255860
255919
  result[i] = '\0';