duckdb 1.4.0 → 1.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (144) hide show
  1. package/package.json +1 -1
  2. package/src/duckdb/extension/core_functions/scalar/generic/current_setting.cpp +1 -4
  3. package/src/duckdb/extension/icu/icu-strptime.cpp +2 -1
  4. package/src/duckdb/extension/json/include/json_common.hpp +2 -4
  5. package/src/duckdb/extension/json/json_functions.cpp +5 -1
  6. package/src/duckdb/extension/parquet/column_writer.cpp +31 -21
  7. package/src/duckdb/extension/parquet/geo_parquet.cpp +21 -6
  8. package/src/duckdb/extension/parquet/include/column_writer.hpp +2 -2
  9. package/src/duckdb/extension/parquet/include/geo_parquet.hpp +28 -1
  10. package/src/duckdb/extension/parquet/include/parquet_writer.hpp +7 -2
  11. package/src/duckdb/extension/parquet/include/reader/string_column_reader.hpp +13 -0
  12. package/src/duckdb/extension/parquet/include/writer/array_column_writer.hpp +4 -0
  13. package/src/duckdb/extension/parquet/parquet_extension.cpp +56 -1
  14. package/src/duckdb/extension/parquet/parquet_reader.cpp +4 -1
  15. package/src/duckdb/extension/parquet/parquet_statistics.cpp +5 -7
  16. package/src/duckdb/extension/parquet/parquet_writer.cpp +15 -8
  17. package/src/duckdb/extension/parquet/reader/string_column_reader.cpp +17 -4
  18. package/src/duckdb/extension/parquet/writer/array_column_writer.cpp +22 -28
  19. package/src/duckdb/extension/parquet/writer/primitive_column_writer.cpp +17 -5
  20. package/src/duckdb/extension/parquet/writer/struct_column_writer.cpp +3 -2
  21. package/src/duckdb/src/catalog/catalog_search_path.cpp +2 -2
  22. package/src/duckdb/src/catalog/catalog_set.cpp +1 -2
  23. package/src/duckdb/src/common/enum_util.cpp +20 -0
  24. package/src/duckdb/src/common/file_system.cpp +0 -30
  25. package/src/duckdb/src/common/sorting/sort.cpp +25 -6
  26. package/src/duckdb/src/common/sorting/sorted_run_merger.cpp +1 -0
  27. package/src/duckdb/src/common/string_util.cpp +24 -0
  28. package/src/duckdb/src/common/virtual_file_system.cpp +59 -10
  29. package/src/duckdb/src/execution/index/art/art_merger.cpp +0 -3
  30. package/src/duckdb/src/execution/index/art/prefix.cpp +4 -0
  31. package/src/duckdb/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +1 -1
  32. package/src/duckdb/src/execution/operator/helper/physical_reset.cpp +2 -2
  33. package/src/duckdb/src/execution/operator/schema/physical_attach.cpp +1 -1
  34. package/src/duckdb/src/execution/physical_plan/plan_asof_join.cpp +3 -3
  35. package/src/duckdb/src/function/table/system/duckdb_connection_count.cpp +45 -0
  36. package/src/duckdb/src/function/table/system/duckdb_settings.cpp +11 -1
  37. package/src/duckdb/src/function/table/system_functions.cpp +1 -0
  38. package/src/duckdb/src/function/table/version/pragma_version.cpp +3 -3
  39. package/src/duckdb/src/include/duckdb/common/enum_util.hpp +8 -0
  40. package/src/duckdb/src/include/duckdb/common/string_util.hpp +2 -0
  41. package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp +4 -1
  42. package/src/duckdb/src/include/duckdb/function/scalar/variant_functions.hpp +1 -1
  43. package/src/duckdb/src/include/duckdb/function/table/system_functions.hpp +4 -0
  44. package/src/duckdb/src/include/duckdb/logging/log_storage.hpp +6 -6
  45. package/src/duckdb/src/include/duckdb/logging/log_type.hpp +26 -3
  46. package/src/duckdb/src/include/duckdb/main/attached_database.hpp +4 -0
  47. package/src/duckdb/src/include/duckdb/main/client_context.hpp +2 -0
  48. package/src/duckdb/src/include/duckdb/main/connection.hpp +0 -1
  49. package/src/duckdb/src/include/duckdb/main/connection_manager.hpp +0 -1
  50. package/src/duckdb/src/include/duckdb/main/database_file_path_manager.hpp +12 -1
  51. package/src/duckdb/src/include/duckdb/main/database_manager.hpp +3 -0
  52. package/src/duckdb/src/include/duckdb/main/relation/create_table_relation.hpp +2 -0
  53. package/src/duckdb/src/include/duckdb/main/relation/create_view_relation.hpp +2 -0
  54. package/src/duckdb/src/include/duckdb/main/relation/delete_relation.hpp +2 -0
  55. package/src/duckdb/src/include/duckdb/main/relation/explain_relation.hpp +2 -0
  56. package/src/duckdb/src/include/duckdb/main/relation/insert_relation.hpp +2 -0
  57. package/src/duckdb/src/include/duckdb/main/relation/query_relation.hpp +1 -0
  58. package/src/duckdb/src/include/duckdb/main/relation/update_relation.hpp +2 -0
  59. package/src/duckdb/src/include/duckdb/main/relation/write_csv_relation.hpp +2 -0
  60. package/src/duckdb/src/include/duckdb/main/relation/write_parquet_relation.hpp +2 -0
  61. package/src/duckdb/src/include/duckdb/main/relation.hpp +2 -1
  62. package/src/duckdb/src/include/duckdb/main/secret/secret.hpp +3 -1
  63. package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp +3 -2
  64. package/src/duckdb/src/include/duckdb/planner/binder.hpp +62 -3
  65. package/src/duckdb/src/include/duckdb/planner/expression_binder/lateral_binder.hpp +2 -2
  66. package/src/duckdb/src/include/duckdb/planner/operator/logical_cte.hpp +1 -1
  67. package/src/duckdb/src/include/duckdb/planner/operator/logical_dependent_join.hpp +3 -3
  68. package/src/duckdb/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp +2 -2
  69. package/src/duckdb/src/include/duckdb/planner/subquery/has_correlated_expressions.hpp +2 -2
  70. package/src/duckdb/src/include/duckdb/planner/subquery/rewrite_cte_scan.hpp +2 -2
  71. package/src/duckdb/src/include/duckdb/planner/tableref/bound_joinref.hpp +1 -1
  72. package/src/duckdb/src/include/duckdb/storage/compression/alp/alp_analyze.hpp +6 -1
  73. package/src/duckdb/src/include/duckdb/storage/compression/alprd/alprd_analyze.hpp +5 -1
  74. package/src/duckdb/src/include/duckdb/storage/metadata/metadata_manager.hpp +9 -7
  75. package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp +2 -0
  76. package/src/duckdb/src/include/duckdb/storage/table/array_column_data.hpp +4 -4
  77. package/src/duckdb/src/include/duckdb/storage/table/column_data.hpp +6 -6
  78. package/src/duckdb/src/include/duckdb/storage/table/list_column_data.hpp +4 -4
  79. package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +4 -4
  80. package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +5 -3
  81. package/src/duckdb/src/include/duckdb/storage/table/row_id_column_data.hpp +4 -4
  82. package/src/duckdb/src/include/duckdb/storage/table/standard_column_data.hpp +4 -4
  83. package/src/duckdb/src/include/duckdb/storage/table/struct_column_data.hpp +4 -4
  84. package/src/duckdb/src/include/duckdb/storage/table/update_segment.hpp +2 -2
  85. package/src/duckdb/src/include/duckdb/transaction/duck_transaction.hpp +2 -1
  86. package/src/duckdb/src/include/duckdb/transaction/update_info.hpp +4 -1
  87. package/src/duckdb/src/include/duckdb/transaction/wal_write_state.hpp +1 -1
  88. package/src/duckdb/src/logging/log_types.cpp +63 -0
  89. package/src/duckdb/src/main/attached_database.cpp +16 -3
  90. package/src/duckdb/src/main/client_context.cpp +27 -19
  91. package/src/duckdb/src/main/connection.cpp +2 -5
  92. package/src/duckdb/src/main/database_file_path_manager.cpp +23 -6
  93. package/src/duckdb/src/main/database_manager.cpp +18 -3
  94. package/src/duckdb/src/main/http/http_util.cpp +3 -1
  95. package/src/duckdb/src/main/relation/create_table_relation.cpp +8 -0
  96. package/src/duckdb/src/main/relation/create_view_relation.cpp +8 -0
  97. package/src/duckdb/src/main/relation/delete_relation.cpp +8 -0
  98. package/src/duckdb/src/main/relation/explain_relation.cpp +8 -0
  99. package/src/duckdb/src/main/relation/insert_relation.cpp +8 -0
  100. package/src/duckdb/src/main/relation/query_relation.cpp +4 -0
  101. package/src/duckdb/src/main/relation/update_relation.cpp +8 -0
  102. package/src/duckdb/src/main/relation/write_csv_relation.cpp +8 -0
  103. package/src/duckdb/src/main/relation/write_parquet_relation.cpp +8 -0
  104. package/src/duckdb/src/main/relation.cpp +2 -2
  105. package/src/duckdb/src/optimizer/filter_combiner.cpp +7 -0
  106. package/src/duckdb/src/optimizer/filter_pushdown.cpp +9 -3
  107. package/src/duckdb/src/optimizer/pushdown/pushdown_get.cpp +4 -1
  108. package/src/duckdb/src/optimizer/rule/comparison_simplification.cpp +3 -7
  109. package/src/duckdb/src/parser/statement/relation_statement.cpp +1 -4
  110. package/src/duckdb/src/parser/transform/statement/transform_create_function.cpp +2 -0
  111. package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp +8 -6
  112. package/src/duckdb/src/planner/binder/statement/bind_create.cpp +1 -5
  113. package/src/duckdb/src/planner/binder/statement/bind_merge_into.cpp +10 -2
  114. package/src/duckdb/src/planner/binder/statement/bind_pragma.cpp +20 -3
  115. package/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp +8 -3
  116. package/src/duckdb/src/planner/binder/tableref/bind_table_function.cpp +9 -2
  117. package/src/duckdb/src/planner/binder.cpp +2 -2
  118. package/src/duckdb/src/planner/expression_binder/lateral_binder.cpp +9 -13
  119. package/src/duckdb/src/planner/expression_binder/table_function_binder.cpp +4 -0
  120. package/src/duckdb/src/planner/expression_binder.cpp +3 -1
  121. package/src/duckdb/src/planner/operator/logical_dependent_join.cpp +2 -2
  122. package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp +12 -14
  123. package/src/duckdb/src/planner/subquery/has_correlated_expressions.cpp +1 -1
  124. package/src/duckdb/src/planner/subquery/rewrite_cte_scan.cpp +2 -2
  125. package/src/duckdb/src/storage/compression/bitpacking.cpp +1 -2
  126. package/src/duckdb/src/storage/data_table.cpp +2 -2
  127. package/src/duckdb/src/storage/local_storage.cpp +1 -1
  128. package/src/duckdb/src/storage/metadata/metadata_manager.cpp +67 -25
  129. package/src/duckdb/src/storage/statistics/string_stats.cpp +8 -0
  130. package/src/duckdb/src/storage/table/array_column_data.cpp +6 -5
  131. package/src/duckdb/src/storage/table/column_data.cpp +23 -9
  132. package/src/duckdb/src/storage/table/column_data_checkpointer.cpp +15 -1
  133. package/src/duckdb/src/storage/table/list_column_data.cpp +5 -4
  134. package/src/duckdb/src/storage/table/row_group.cpp +8 -8
  135. package/src/duckdb/src/storage/table/row_group_collection.cpp +12 -8
  136. package/src/duckdb/src/storage/table/row_id_column_data.cpp +5 -4
  137. package/src/duckdb/src/storage/table/standard_column_data.cpp +9 -8
  138. package/src/duckdb/src/storage/table/struct_column_data.cpp +10 -9
  139. package/src/duckdb/src/storage/table/update_segment.cpp +12 -10
  140. package/src/duckdb/src/transaction/commit_state.cpp +18 -0
  141. package/src/duckdb/src/transaction/duck_transaction.cpp +2 -10
  142. package/src/duckdb/src/transaction/wal_write_state.cpp +5 -5
  143. package/src/duckdb/third_party/httplib/httplib.hpp +6 -1
  144. package/src/duckdb/ub_src_function_table_system.cpp +2 -0
@@ -24,6 +24,14 @@ BoundStatement InsertRelation::Bind(Binder &binder) {
24
24
  return binder.Bind(stmt.Cast<SQLStatement>());
25
25
  }
26
26
 
27
+ unique_ptr<QueryNode> InsertRelation::GetQueryNode() {
28
+ throw InternalException("Cannot create a query node from an insert relation");
29
+ }
30
+
31
+ string InsertRelation::GetQuery() {
32
+ return string();
33
+ }
34
+
27
35
  const vector<ColumnDefinition> &InsertRelation::Columns() {
28
36
  return columns;
29
37
  }
@@ -49,6 +49,10 @@ unique_ptr<QueryNode> QueryRelation::GetQueryNode() {
49
49
  return std::move(select->node);
50
50
  }
51
51
 
52
+ string QueryRelation::GetQuery() {
53
+ return query;
54
+ }
55
+
52
56
  unique_ptr<TableRef> QueryRelation::GetTableRef() {
53
57
  auto subquery_ref = make_uniq<SubqueryRef>(GetSelectStatement(), GetAlias());
54
58
  return std::move(subquery_ref);
@@ -35,6 +35,14 @@ BoundStatement UpdateRelation::Bind(Binder &binder) {
35
35
  return binder.Bind(stmt.Cast<SQLStatement>());
36
36
  }
37
37
 
38
+ unique_ptr<QueryNode> UpdateRelation::GetQueryNode() {
39
+ throw InternalException("Cannot create a query node from an update relation");
40
+ }
41
+
42
+ string UpdateRelation::GetQuery() {
43
+ return string();
44
+ }
45
+
38
46
  const vector<ColumnDefinition> &UpdateRelation::Columns() {
39
47
  return columns;
40
48
  }
@@ -25,6 +25,14 @@ BoundStatement WriteCSVRelation::Bind(Binder &binder) {
25
25
  return binder.Bind(copy.Cast<SQLStatement>());
26
26
  }
27
27
 
28
+ unique_ptr<QueryNode> WriteCSVRelation::GetQueryNode() {
29
+ throw InternalException("Cannot create a query node from a write CSV relation");
30
+ }
31
+
32
+ string WriteCSVRelation::GetQuery() {
33
+ return string();
34
+ }
35
+
28
36
  const vector<ColumnDefinition> &WriteCSVRelation::Columns() {
29
37
  return columns;
30
38
  }
@@ -25,6 +25,14 @@ BoundStatement WriteParquetRelation::Bind(Binder &binder) {
25
25
  return binder.Bind(copy.Cast<SQLStatement>());
26
26
  }
27
27
 
28
+ unique_ptr<QueryNode> WriteParquetRelation::GetQueryNode() {
29
+ throw InternalException("Cannot create a query node from a write parquet relation");
30
+ }
31
+
32
+ string WriteParquetRelation::GetQuery() {
33
+ return string();
34
+ }
35
+
28
36
  const vector<ColumnDefinition> &WriteParquetRelation::Columns() {
29
37
  return columns;
30
38
  }
@@ -394,8 +394,8 @@ string Relation::ToString() {
394
394
  }
395
395
 
396
396
  // LCOV_EXCL_START
397
- unique_ptr<QueryNode> Relation::GetQueryNode() {
398
- throw InternalException("Cannot create a query node from this node type");
397
+ string Relation::GetQuery() {
398
+ return GetQueryNode()->ToString();
399
399
  }
400
400
 
401
401
  void Relation::Head(idx_t limit) {
@@ -1,5 +1,6 @@
1
1
  #include "duckdb/optimizer/filter_combiner.hpp"
2
2
 
3
+ #include "duckdb/common/enums/expression_type.hpp"
3
4
  #include "duckdb/execution/expression_executor.hpp"
4
5
  #include "duckdb/optimizer/optimizer.hpp"
5
6
  #include "duckdb/planner/expression.hpp"
@@ -907,6 +908,12 @@ FilterResult FilterCombiner::AddTransitiveFilters(BoundComparisonExpression &com
907
908
  idx_t left_equivalence_set = GetEquivalenceSet(left_node);
908
909
  idx_t right_equivalence_set = GetEquivalenceSet(right_node);
909
910
  if (left_equivalence_set == right_equivalence_set) {
911
+ if (comparison.GetExpressionType() == ExpressionType::COMPARE_GREATERTHAN ||
912
+ comparison.GetExpressionType() == ExpressionType::COMPARE_LESSTHAN) {
913
+ // non equal comparison has equal equivalence set, then it is unsatisfiable
914
+ // e.g., j > i AND i < j is unsatisfiable
915
+ return FilterResult::UNSATISFIABLE;
916
+ }
910
917
  // this equality filter already exists, prune it
911
918
  return FilterResult::SUCCESS;
912
919
  }
@@ -207,17 +207,23 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownJoin(unique_ptr<LogicalOpera
207
207
  }
208
208
  return result;
209
209
  }
210
- void FilterPushdown::PushFilters() {
210
+ FilterResult FilterPushdown::PushFilters() {
211
211
  for (auto &f : filters) {
212
212
  auto result = combiner.AddFilter(std::move(f->filter));
213
213
  D_ASSERT(result != FilterResult::UNSUPPORTED);
214
- (void)result;
214
+ if (result == FilterResult::UNSATISFIABLE) {
215
+ // one of the filters is unsatisfiable - abort filter pushdown
216
+ return FilterResult::UNSATISFIABLE;
217
+ }
215
218
  }
216
219
  filters.clear();
220
+ return FilterResult::SUCCESS;
217
221
  }
218
222
 
219
223
  FilterResult FilterPushdown::AddFilter(unique_ptr<Expression> expr) {
220
- PushFilters();
224
+ if (PushFilters() == FilterResult::UNSATISFIABLE) {
225
+ return FilterResult::UNSATISFIABLE;
226
+ }
221
227
  // split up the filters by AND predicate
222
228
  vector<unique_ptr<Expression>> expressions;
223
229
  expressions.push_back(std::move(expr));
@@ -4,6 +4,7 @@
4
4
  #include "duckdb/planner/expression/bound_parameter_expression.hpp"
5
5
  #include "duckdb/planner/operator/logical_filter.hpp"
6
6
  #include "duckdb/planner/operator/logical_get.hpp"
7
+ #include "duckdb/planner/operator/logical_empty_result.hpp"
7
8
 
8
9
  namespace duckdb {
9
10
  unique_ptr<LogicalOperator> FilterPushdown::PushdownGet(unique_ptr<LogicalOperator> op) {
@@ -48,7 +49,9 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownGet(unique_ptr<LogicalOperat
48
49
  // the table function does not support filter pushdown: push a LogicalFilter on top
49
50
  return FinishPushdown(std::move(op));
50
51
  }
51
- PushFilters();
52
+ if (PushFilters() == FilterResult::UNSATISFIABLE) {
53
+ return make_uniq<LogicalEmptyResult>(std::move(op));
54
+ }
52
55
 
53
56
  //! We generate the table filters that will be executed during the table scan
54
57
  vector<FilterPushdownResult> pushdown_results;
@@ -56,13 +56,8 @@ unique_ptr<Expression> ComparisonSimplificationRule::Apply(LogicalOperator &op,
56
56
  // Is the constant cast invertible?
57
57
  if (!cast_constant.IsNull() &&
58
58
  !BoundCastExpression::CastIsInvertible(cast_expression.return_type, target_type)) {
59
- // Is it actually invertible?
60
- Value uncast_constant;
61
- if (!cast_constant.TryCastAs(rewriter.context, constant_value.type(), uncast_constant, &error_message,
62
- true) ||
63
- uncast_constant != constant_value) {
64
- return nullptr;
65
- }
59
+ // Cast is not invertible, so we do not rewrite this expression to ensure that the cast is executed
60
+ return nullptr;
66
61
  }
67
62
 
68
63
  //! We can cast, now we change our column_ref_expression from an operator cast to a column reference
@@ -75,6 +70,7 @@ unique_ptr<Expression> ComparisonSimplificationRule::Apply(LogicalOperator &op,
75
70
  expr.left = std::move(new_constant_expr);
76
71
  expr.right = std::move(child_expression);
77
72
  }
73
+ changes_made = true;
78
74
  }
79
75
  return nullptr;
80
76
  }
@@ -5,10 +5,7 @@ namespace duckdb {
5
5
 
6
6
  RelationStatement::RelationStatement(shared_ptr<Relation> relation_p)
7
7
  : SQLStatement(StatementType::RELATION_STATEMENT), relation(std::move(relation_p)) {
8
- if (relation->type == RelationType::QUERY_RELATION) {
9
- auto &query_relation = relation->Cast<QueryRelation>();
10
- query = query_relation.query;
11
- }
8
+ query = relation->GetQuery();
12
9
  }
13
10
 
14
11
  unique_ptr<SQLStatement> RelationStatement::Copy() const {
@@ -47,6 +47,8 @@ unique_ptr<MacroFunction> Transformer::TransformMacroFunction(duckdb_libpgquery:
47
47
  default_expr = make_uniq<ConstantExpression>(std::move(default_value));
48
48
  default_expr->SetAlias(param.name);
49
49
  macro_func->default_parameters[param.name] = std::move(default_expr);
50
+ } else if (!macro_func->default_parameters.empty()) {
51
+ throw ParserException("Parameter without a default follows parameter with a default");
50
52
  }
51
53
  }
52
54
 
@@ -186,9 +186,10 @@ static unique_ptr<Expression> PlanUncorrelatedSubquery(Binder &binder, BoundSubq
186
186
  }
187
187
  }
188
188
 
189
- static unique_ptr<LogicalDependentJoin>
190
- CreateDuplicateEliminatedJoin(const vector<CorrelatedColumnInfo> &correlated_columns, JoinType join_type,
191
- unique_ptr<LogicalOperator> original_plan, bool perform_delim) {
189
+ static unique_ptr<LogicalDependentJoin> CreateDuplicateEliminatedJoin(const CorrelatedColumns &correlated_columns,
190
+ JoinType join_type,
191
+ unique_ptr<LogicalOperator> original_plan,
192
+ bool perform_delim) {
192
193
  auto delim_join = make_uniq<LogicalDependentJoin>(join_type);
193
194
  delim_join->correlated_columns = correlated_columns;
194
195
  delim_join->perform_delim = perform_delim;
@@ -216,7 +217,7 @@ static bool PerformDelimOnType(const LogicalType &type) {
216
217
  return true;
217
218
  }
218
219
 
219
- static bool PerformDuplicateElimination(Binder &binder, vector<CorrelatedColumnInfo> &correlated_columns) {
220
+ static bool PerformDuplicateElimination(Binder &binder, CorrelatedColumns &correlated_columns) {
220
221
  if (!ClientConfig::GetConfig(binder.context).enable_optimizer) {
221
222
  // if optimizations are disabled we always do a delim join
222
223
  return true;
@@ -235,7 +236,8 @@ static bool PerformDuplicateElimination(Binder &binder, vector<CorrelatedColumnI
235
236
  auto type = LogicalType::BIGINT;
236
237
  auto name = "delim_index";
237
238
  CorrelatedColumnInfo info(binding, type, name, 0);
238
- correlated_columns.insert(correlated_columns.begin(), std::move(info));
239
+ correlated_columns.AddColumn(std::move(info));
240
+ correlated_columns.SetDelimIndexToZero();
239
241
  return false;
240
242
  }
241
243
 
@@ -403,7 +405,7 @@ void Binder::PlanSubqueries(unique_ptr<Expression> &expr_ptr, unique_ptr<Logical
403
405
  }
404
406
 
405
407
  unique_ptr<LogicalOperator> Binder::PlanLateralJoin(unique_ptr<LogicalOperator> left, unique_ptr<LogicalOperator> right,
406
- vector<CorrelatedColumnInfo> &correlated, JoinType join_type,
408
+ CorrelatedColumns &correlated, JoinType join_type,
407
409
  unique_ptr<Expression> condition) {
408
410
  // scan the right operator for correlated columns
409
411
  // correlated LATERAL JOIN
@@ -345,11 +345,7 @@ SchemaCatalogEntry &Binder::BindCreateFunctionInfo(CreateInfo &info) {
345
345
  try {
346
346
  dummy_binder->Bind(*query_node);
347
347
  } catch (const std::exception &ex) {
348
- // TODO: we would like to do something like "error = ErrorData(ex);" here,
349
- // but that breaks macro's like "create macro m(x) as table (from query_table(x));",
350
- // because dummy-binding these always throws an error instead of a ParameterNotResolvedException.
351
- // So, for now, we allow macro's with bind errors to be created.
352
- // Binding is still useful because we can create the dependencies.
348
+ error = ErrorData(ex);
353
349
  }
354
350
  }
355
351
 
@@ -232,10 +232,18 @@ BoundStatement Binder::Bind(MergeIntoStatement &stmt) {
232
232
  auto bound_join_node = Bind(join);
233
233
 
234
234
  auto root = CreatePlan(*bound_join_node);
235
+ auto join_ref = reference<LogicalOperator>(*root);
236
+ while (join_ref.get().children.size() == 1) {
237
+ join_ref = *join_ref.get().children[0];
238
+ }
239
+ if (join_ref.get().children.size() != 2) {
240
+ throw NotImplementedException("Expected a join after binding a join operator - but got a %s",
241
+ join_ref.get().type);
242
+ }
235
243
  // kind of hacky, CreatePlan turns a RIGHT join into a LEFT join so the children get reversed from what we need
236
244
  bool inverted = join.type == JoinType::RIGHT;
237
- auto &source = root->children[inverted ? 1 : 0];
238
- auto &get = root->children[inverted ? 0 : 1]->Cast<LogicalGet>();
245
+ auto &source = join_ref.get().children[inverted ? 1 : 0];
246
+ auto &get = join_ref.get().children[inverted ? 0 : 1]->Cast<LogicalGet>();
239
247
 
240
248
  auto merge_into = make_uniq<LogicalMergeInto>(table);
241
249
  merge_into->table_index = GenerateTableIndex();
@@ -2,6 +2,7 @@
2
2
  #include "duckdb/parser/statement/pragma_statement.hpp"
3
3
  #include "duckdb/planner/operator/logical_pragma.hpp"
4
4
  #include "duckdb/catalog/catalog_entry/pragma_function_catalog_entry.hpp"
5
+ #include "duckdb/catalog/catalog_entry/table_function_catalog_entry.hpp"
5
6
  #include "duckdb/catalog/catalog.hpp"
6
7
  #include "duckdb/function/function_binder.hpp"
7
8
  #include "duckdb/planner/expression_binder/constant_binder.hpp"
@@ -28,16 +29,32 @@ unique_ptr<BoundPragmaInfo> Binder::BindPragma(PragmaInfo &info, QueryErrorConte
28
29
  }
29
30
 
30
31
  // bind the pragma function
31
- auto &entry = Catalog::GetEntry<PragmaFunctionCatalogEntry>(context, INVALID_CATALOG, DEFAULT_SCHEMA, info.name);
32
+ auto entry = Catalog::GetEntry<PragmaFunctionCatalogEntry>(context, INVALID_CATALOG, DEFAULT_SCHEMA, info.name,
33
+ OnEntryNotFound::RETURN_NULL);
34
+ if (!entry) {
35
+ // try to find whether a table extry might exist
36
+ auto table_entry = Catalog::GetEntry<TableFunctionCatalogEntry>(context, INVALID_CATALOG, DEFAULT_SCHEMA,
37
+ info.name, OnEntryNotFound::RETURN_NULL);
38
+ if (table_entry) {
39
+ // there is a table entry with the same name, now throw more explicit error message
40
+ throw CatalogException("Pragma Function with name %s does not exist, but a table function with the same "
41
+ "name exists, try `CALL %s(...)`",
42
+ info.name, info.name);
43
+ }
44
+ // rebind to throw exception
45
+ entry = Catalog::GetEntry<PragmaFunctionCatalogEntry>(context, INVALID_CATALOG, DEFAULT_SCHEMA, info.name,
46
+ OnEntryNotFound::THROW_EXCEPTION);
47
+ }
48
+
32
49
  FunctionBinder function_binder(*this);
33
50
  ErrorData error;
34
- auto bound_idx = function_binder.BindFunction(entry.name, entry.functions, params, error);
51
+ auto bound_idx = function_binder.BindFunction(entry->name, entry->functions, params, error);
35
52
  if (!bound_idx.IsValid()) {
36
53
  D_ASSERT(error.HasError());
37
54
  error.AddQueryLocation(error_context);
38
55
  error.Throw();
39
56
  }
40
- auto bound_function = entry.functions.GetFunctionByOffset(bound_idx.GetIndex());
57
+ auto bound_function = entry->functions.GetFunctionByOffset(bound_idx.GetIndex());
41
58
  // bind and check named params
42
59
  BindNamedParameters(bound_function.named_parameters, named_parameters, error_context, bound_function.name);
43
60
  return make_uniq<BoundPragmaInfo>(std::move(bound_function), std::move(params), std::move(named_parameters));
@@ -58,10 +58,15 @@ static void ConstructPivots(PivotRef &ref, vector<PivotValueElement> &pivot_valu
58
58
  }
59
59
  }
60
60
 
61
- static void ExtractPivotExpressions(ParsedExpression &root_expr, case_insensitive_set_t &handled_columns) {
61
+ static void ExtractPivotExpressions(ParsedExpression &root_expr, case_insensitive_set_t &handled_columns,
62
+ optional_ptr<DummyBinding> macro_binding) {
62
63
  ParsedExpressionIterator::VisitExpression<ColumnRefExpression>(
63
64
  root_expr, [&](const ColumnRefExpression &child_colref) {
64
65
  if (child_colref.IsQualified()) {
66
+ if (child_colref.column_names[0].find(DummyBinding::DUMMY_NAME) != string::npos && macro_binding &&
67
+ macro_binding->HasMatchingBinding(child_colref.GetName())) {
68
+ throw ParameterNotResolvedException();
69
+ }
65
70
  throw BinderException(child_colref, "PIVOT expression cannot contain qualified columns");
66
71
  }
67
72
  handled_columns.insert(child_colref.GetColumnName());
@@ -492,7 +497,7 @@ unique_ptr<SelectNode> Binder::BindPivot(PivotRef &ref, vector<unique_ptr<Parsed
492
497
  }
493
498
  }
494
499
  for (auto &aggr : pivot_aggregates) {
495
- ExtractPivotExpressions(aggr.get(), handled_columns);
500
+ ExtractPivotExpressions(aggr.get(), handled_columns, macro_binding);
496
501
  }
497
502
 
498
503
  // first add all pivots to the set of handled columns, and check for duplicates
@@ -521,7 +526,7 @@ unique_ptr<SelectNode> Binder::BindPivot(PivotRef &ref, vector<unique_ptr<Parsed
521
526
  total_pivots *= pivot.entries.size();
522
527
  // add the pivoted column to the columns that have been handled
523
528
  for (auto &pivot_name : pivot.pivot_expressions) {
524
- ExtractPivotExpressions(*pivot_name, handled_columns);
529
+ ExtractPivotExpressions(*pivot_name, handled_columns, macro_binding);
525
530
  }
526
531
  value_set_t pivots;
527
532
  for (auto &entry : pivot.entries) {
@@ -469,8 +469,15 @@ unique_ptr<BoundTableRef> Binder::Bind(TableFunctionRef &ref) {
469
469
  }
470
470
  }
471
471
 
472
- auto get = BindTableFunctionInternal(table_function, ref, std::move(parameters), std::move(named_parameters),
473
- std::move(input_table_types), std::move(input_table_names));
472
+ unique_ptr<LogicalOperator> get;
473
+ try {
474
+ get = BindTableFunctionInternal(table_function, ref, std::move(parameters), std::move(named_parameters),
475
+ std::move(input_table_types), std::move(input_table_names));
476
+ } catch (std::exception &ex) {
477
+ error = ErrorData(ex);
478
+ error.AddQueryLocation(ref);
479
+ error.Throw();
480
+ }
474
481
  auto table_function_ref = make_uniq<BoundTableFunction>(std::move(get));
475
482
  table_function_ref->subquery = std::move(subquery);
476
483
  return std::move(table_function_ref);
@@ -440,7 +440,7 @@ void Binder::MoveCorrelatedExpressions(Binder &other) {
440
440
  other.correlated_columns.clear();
441
441
  }
442
442
 
443
- void Binder::MergeCorrelatedColumns(vector<CorrelatedColumnInfo> &other) {
443
+ void Binder::MergeCorrelatedColumns(CorrelatedColumns &other) {
444
444
  for (idx_t i = 0; i < other.size(); i++) {
445
445
  AddCorrelatedColumn(other[i]);
446
446
  }
@@ -449,7 +449,7 @@ void Binder::MergeCorrelatedColumns(vector<CorrelatedColumnInfo> &other) {
449
449
  void Binder::AddCorrelatedColumn(const CorrelatedColumnInfo &info) {
450
450
  // we only add correlated columns to the list if they are not already there
451
451
  if (std::find(correlated_columns.begin(), correlated_columns.end(), info) == correlated_columns.end()) {
452
- correlated_columns.push_back(info);
452
+ correlated_columns.AddColumn(info);
453
453
  }
454
454
  }
455
455
 
@@ -17,7 +17,7 @@ void LateralBinder::ExtractCorrelatedColumns(Expression &expr) {
17
17
  // add the correlated column info
18
18
  CorrelatedColumnInfo info(bound_colref);
19
19
  if (std::find(correlated_columns.begin(), correlated_columns.end(), info) == correlated_columns.end()) {
20
- correlated_columns.push_back(std::move(info));
20
+ correlated_columns.AddColumn(std::move(info)); // TODO is adding to the front OK here?
21
21
  }
22
22
  }
23
23
  }
@@ -54,8 +54,7 @@ string LateralBinder::UnsupportedAggregateMessage() {
54
54
  return "LATERAL join cannot contain aggregates!";
55
55
  }
56
56
 
57
- static void ReduceColumnRefDepth(BoundColumnRefExpression &expr,
58
- const vector<CorrelatedColumnInfo> &correlated_columns) {
57
+ static void ReduceColumnRefDepth(BoundColumnRefExpression &expr, const CorrelatedColumns &correlated_columns) {
59
58
  // don't need to reduce this
60
59
  if (expr.depth == 0) {
61
60
  return;
@@ -69,8 +68,7 @@ static void ReduceColumnRefDepth(BoundColumnRefExpression &expr,
69
68
  }
70
69
  }
71
70
 
72
- static void ReduceColumnDepth(vector<CorrelatedColumnInfo> &columns,
73
- const vector<CorrelatedColumnInfo> &affected_columns) {
71
+ static void ReduceColumnDepth(CorrelatedColumns &columns, const CorrelatedColumns &affected_columns) {
74
72
  for (auto &s_correlated : columns) {
75
73
  for (auto &affected : affected_columns) {
76
74
  if (affected == s_correlated) {
@@ -83,8 +81,7 @@ static void ReduceColumnDepth(vector<CorrelatedColumnInfo> &columns,
83
81
 
84
82
  class ExpressionDepthReducerRecursive : public BoundNodeVisitor {
85
83
  public:
86
- explicit ExpressionDepthReducerRecursive(const vector<CorrelatedColumnInfo> &correlated)
87
- : correlated_columns(correlated) {
84
+ explicit ExpressionDepthReducerRecursive(const CorrelatedColumns &correlated) : correlated_columns(correlated) {
88
85
  }
89
86
 
90
87
  void VisitExpression(unique_ptr<Expression> &expression) override {
@@ -106,20 +103,19 @@ public:
106
103
  BoundNodeVisitor::VisitBoundTableRef(ref);
107
104
  }
108
105
 
109
- static void ReduceExpressionSubquery(BoundSubqueryExpression &expr,
110
- const vector<CorrelatedColumnInfo> &correlated_columns) {
106
+ static void ReduceExpressionSubquery(BoundSubqueryExpression &expr, const CorrelatedColumns &correlated_columns) {
111
107
  ReduceColumnDepth(expr.binder->correlated_columns, correlated_columns);
112
108
  ExpressionDepthReducerRecursive recursive(correlated_columns);
113
109
  recursive.VisitBoundQueryNode(*expr.subquery);
114
110
  }
115
111
 
116
112
  private:
117
- const vector<CorrelatedColumnInfo> &correlated_columns;
113
+ const CorrelatedColumns &correlated_columns;
118
114
  };
119
115
 
120
116
  class ExpressionDepthReducer : public LogicalOperatorVisitor {
121
117
  public:
122
- explicit ExpressionDepthReducer(const vector<CorrelatedColumnInfo> &correlated) : correlated_columns(correlated) {
118
+ explicit ExpressionDepthReducer(const CorrelatedColumns &correlated) : correlated_columns(correlated) {
123
119
  }
124
120
 
125
121
  protected:
@@ -133,10 +129,10 @@ protected:
133
129
  return nullptr;
134
130
  }
135
131
 
136
- const vector<CorrelatedColumnInfo> &correlated_columns;
132
+ const CorrelatedColumns &correlated_columns;
137
133
  };
138
134
 
139
- void LateralBinder::ReduceExpressionDepth(LogicalOperator &op, const vector<CorrelatedColumnInfo> &correlated) {
135
+ void LateralBinder::ReduceExpressionDepth(LogicalOperator &op, const CorrelatedColumns &correlated) {
140
136
  ExpressionDepthReducer depth_reducer(correlated);
141
137
  depth_reducer.VisitOperator(op);
142
138
  }
@@ -27,9 +27,13 @@ BindResult TableFunctionBinder::BindColumnReference(unique_ptr<ParsedExpression>
27
27
  if (lambda_ref) {
28
28
  return BindLambdaReference(lambda_ref->Cast<LambdaRefExpression>(), depth);
29
29
  }
30
+
30
31
  if (binder.macro_binding && binder.macro_binding->HasMatchingBinding(col_ref.GetName())) {
31
32
  throw ParameterNotResolvedException();
32
33
  }
34
+ } else if (col_ref.column_names[0].find(DummyBinding::DUMMY_NAME) != string::npos && binder.macro_binding &&
35
+ binder.macro_binding->HasMatchingBinding(col_ref.GetName())) {
36
+ throw ParameterNotResolvedException();
33
37
  }
34
38
 
35
39
  auto query_location = col_ref.GetQueryLocation();
@@ -103,7 +103,9 @@ BindResult ExpressionBinder::BindExpression(unique_ptr<ParsedExpression> &expr,
103
103
  case ExpressionClass::STAR:
104
104
  return BindResult(BinderException::Unsupported(expr_ref, "STAR expression is not supported here"));
105
105
  default:
106
- throw NotImplementedException("Unimplemented expression class");
106
+ return BindResult(
107
+ NotImplementedException("Unimplemented expression class in ExpressionBinder::BindExpression: %s",
108
+ EnumUtil::ToString(expr_ref.GetExpressionClass())));
107
109
  }
108
110
  }
109
111
 
@@ -3,7 +3,7 @@
3
3
  namespace duckdb {
4
4
 
5
5
  LogicalDependentJoin::LogicalDependentJoin(unique_ptr<LogicalOperator> left, unique_ptr<LogicalOperator> right,
6
- vector<CorrelatedColumnInfo> correlated_columns, JoinType type,
6
+ CorrelatedColumns correlated_columns, JoinType type,
7
7
  unique_ptr<Expression> condition)
8
8
  : LogicalComparisonJoin(type, LogicalOperatorType::LOGICAL_DEPENDENT_JOIN), join_condition(std::move(condition)),
9
9
  correlated_columns(std::move(correlated_columns)) {
@@ -17,7 +17,7 @@ LogicalDependentJoin::LogicalDependentJoin(JoinType join_type)
17
17
 
18
18
  unique_ptr<LogicalOperator> LogicalDependentJoin::Create(unique_ptr<LogicalOperator> left,
19
19
  unique_ptr<LogicalOperator> right,
20
- vector<CorrelatedColumnInfo> correlated_columns, JoinType type,
20
+ CorrelatedColumns correlated_columns, JoinType type,
21
21
  unique_ptr<Expression> condition) {
22
22
  return make_uniq<LogicalDependentJoin>(std::move(left), std::move(right), std::move(correlated_columns), type,
23
23
  std::move(condition));
@@ -18,9 +18,8 @@
18
18
 
19
19
  namespace duckdb {
20
20
 
21
- FlattenDependentJoins::FlattenDependentJoins(Binder &binder, const vector<CorrelatedColumnInfo> &correlated,
22
- bool perform_delim, bool any_join,
23
- optional_ptr<FlattenDependentJoins> parent)
21
+ FlattenDependentJoins::FlattenDependentJoins(Binder &binder, const CorrelatedColumns &correlated, bool perform_delim,
22
+ bool any_join, optional_ptr<FlattenDependentJoins> parent)
24
23
  : binder(binder), delim_offset(DConstants::INVALID_INDEX), correlated_columns(correlated),
25
24
  perform_delim(perform_delim), any_join(any_join), parent(parent) {
26
25
  for (idx_t i = 0; i < correlated_columns.size(); i++) {
@@ -30,8 +29,7 @@ FlattenDependentJoins::FlattenDependentJoins(Binder &binder, const vector<Correl
30
29
  }
31
30
  }
32
31
 
33
- static void CreateDelimJoinConditions(LogicalComparisonJoin &delim_join,
34
- const vector<CorrelatedColumnInfo> &correlated_columns,
32
+ static void CreateDelimJoinConditions(LogicalComparisonJoin &delim_join, const CorrelatedColumns &correlated_columns,
35
33
  vector<ColumnBinding> bindings, idx_t base_offset, bool perform_delim) {
36
34
  auto col_count = perform_delim ? correlated_columns.size() : 1;
37
35
  for (idx_t i = 0; i < col_count; i++) {
@@ -50,7 +48,7 @@ static void CreateDelimJoinConditions(LogicalComparisonJoin &delim_join,
50
48
 
51
49
  unique_ptr<LogicalOperator> FlattenDependentJoins::DecorrelateIndependent(Binder &binder,
52
50
  unique_ptr<LogicalOperator> plan) {
53
- vector<CorrelatedColumnInfo> correlated;
51
+ CorrelatedColumns correlated;
54
52
  FlattenDependentJoins flatten(binder, correlated);
55
53
  return flatten.Decorrelate(std::move(plan));
56
54
  }
@@ -80,12 +78,12 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::Decorrelate(unique_ptr<Logica
80
78
  entry->second = false;
81
79
 
82
80
  // rewrite
83
- idx_t lateral_depth = 0;
81
+ idx_t next_lateral_depth = 0;
84
82
 
85
- RewriteCorrelatedExpressions rewriter(base_binding, correlated_map, lateral_depth);
83
+ RewriteCorrelatedExpressions rewriter(base_binding, correlated_map, next_lateral_depth);
86
84
  rewriter.VisitOperator(*plan);
87
85
 
88
- RewriteCorrelatedExpressions recursive_rewriter(base_binding, correlated_map, lateral_depth, true);
86
+ RewriteCorrelatedExpressions recursive_rewriter(base_binding, correlated_map, next_lateral_depth, true);
89
87
  recursive_rewriter.VisitOperator(*plan);
90
88
  } else {
91
89
  op.children[0] = Decorrelate(std::move(op.children[0]));
@@ -94,8 +92,8 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::Decorrelate(unique_ptr<Logica
94
92
  if (!op.perform_delim) {
95
93
  // if we are not performing a delim join, we push a row_number() OVER() window operator on the LHS
96
94
  // and perform all duplicate elimination on that row number instead
97
- D_ASSERT(op.correlated_columns[0].type.id() == LogicalTypeId::BIGINT);
98
- auto window = make_uniq<LogicalWindow>(op.correlated_columns[0].binding.table_index);
95
+ const auto &op_col = op.correlated_columns[op.correlated_columns.GetDelimIndex()];
96
+ auto window = make_uniq<LogicalWindow>(op_col.binding.table_index);
99
97
  auto row_number = make_uniq<BoundWindowExpression>(ExpressionType::WINDOW_ROW_NUMBER, LogicalType::BIGINT,
100
98
  nullptr, nullptr);
101
99
  row_number->start = WindowBoundary::UNBOUNDED_PRECEDING;
@@ -114,9 +112,9 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::Decorrelate(unique_ptr<Logica
114
112
  flatten.DetectCorrelatedExpressions(*delim_join->children[1], op.is_lateral_join, lateral_depth);
115
113
 
116
114
  if (delim_join->children[1]->type == LogicalOperatorType::LOGICAL_MATERIALIZED_CTE) {
117
- auto &cte = delim_join->children[1]->Cast<LogicalMaterializedCTE>();
115
+ auto &cte_ref = delim_join->children[1]->Cast<LogicalMaterializedCTE>();
118
116
  // check if the left side of the CTE has correlated expressions
119
- auto entry = flatten.has_correlated_expressions.find(*cte.children[0]);
117
+ auto entry = flatten.has_correlated_expressions.find(*cte_ref.children[0]);
120
118
  if (entry != flatten.has_correlated_expressions.end()) {
121
119
  if (!entry->second) {
122
120
  // the left side of the CTE has no correlated expressions, we can push the DEPENDENT_JOIN down
@@ -132,7 +130,7 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::Decorrelate(unique_ptr<Logica
132
130
  delim_join->children[1] =
133
131
  flatten.PushDownDependentJoin(std::move(delim_join->children[1]), propagate_null_values, lateral_depth);
134
132
  data_offset = flatten.data_offset;
135
- auto left_offset = delim_join->children[0]->GetColumnBindings().size();
133
+ const auto left_offset = delim_join->children[0]->GetColumnBindings().size();
136
134
  if (!parent) {
137
135
  delim_offset = left_offset + flatten.delim_offset;
138
136
  }
@@ -7,7 +7,7 @@
7
7
 
8
8
  namespace duckdb {
9
9
 
10
- HasCorrelatedExpressions::HasCorrelatedExpressions(const vector<CorrelatedColumnInfo> &correlated, bool lateral,
10
+ HasCorrelatedExpressions::HasCorrelatedExpressions(const CorrelatedColumns &correlated, bool lateral,
11
11
  idx_t lateral_depth)
12
12
  : has_correlated_expressions(false), lateral(lateral), correlated_columns(correlated),
13
13
  lateral_depth(lateral_depth) {
@@ -14,7 +14,7 @@
14
14
 
15
15
  namespace duckdb {
16
16
 
17
- RewriteCTEScan::RewriteCTEScan(idx_t table_index, const vector<CorrelatedColumnInfo> &correlated_columns)
17
+ RewriteCTEScan::RewriteCTEScan(idx_t table_index, const CorrelatedColumns &correlated_columns)
18
18
  : table_index(table_index), correlated_columns(correlated_columns) {
19
19
  }
20
20
 
@@ -49,7 +49,7 @@ void RewriteCTEScan::VisitOperator(LogicalOperator &op) {
49
49
  // The correlated columns must be placed at the beginning of the
50
50
  // correlated_columns list. Otherwise, further column accesses
51
51
  // and rewrites will fail.
52
- join.correlated_columns.emplace(join.correlated_columns.begin(), corr);
52
+ join.correlated_columns.AddColumn(std::move(corr));
53
53
  }
54
54
  }
55
55
  }
@@ -341,8 +341,6 @@ unique_ptr<AnalyzeState> BitpackingInitAnalyze(ColumnData &col_data, PhysicalTyp
341
341
 
342
342
  template <class T>
343
343
  bool BitpackingAnalyze(AnalyzeState &state, Vector &input, idx_t count) {
344
- auto &analyze_state = state.Cast<BitpackingAnalyzeState<T>>();
345
-
346
344
  // We use BITPACKING_METADATA_GROUP_SIZE tuples, which can exceed the block size.
347
345
  // In that case, we disable bitpacking.
348
346
  // we are conservative here by multiplying by 2
@@ -351,6 +349,7 @@ bool BitpackingAnalyze(AnalyzeState &state, Vector &input, idx_t count) {
351
349
  return false;
352
350
  }
353
351
 
352
+ auto &analyze_state = state.Cast<BitpackingAnalyzeState<T>>();
354
353
  UnifiedVectorFormat vdata;
355
354
  input.ToUnifiedFormat(count, vdata);
356
355