duckdb 0.7.2-dev614.0 → 0.7.2-dev717.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/package.json +1 -1
  2. package/src/duckdb/extension/json/include/json_functions.hpp +1 -0
  3. package/src/duckdb/extension/json/json_functions.cpp +11 -4
  4. package/src/duckdb/src/catalog/catalog_entry/table_catalog_entry.cpp +3 -3
  5. package/src/duckdb/src/catalog/dependency_list.cpp +12 -0
  6. package/src/duckdb/src/common/string_util.cpp +4 -1
  7. package/src/duckdb/src/common/types/column_data_collection_segment.cpp +11 -6
  8. package/src/duckdb/src/common/vector_operations/vector_cast.cpp +2 -1
  9. package/src/duckdb/src/execution/aggregate_hashtable.cpp +10 -5
  10. package/src/duckdb/src/execution/expression_executor/execute_cast.cpp +2 -1
  11. package/src/duckdb/src/execution/index/art/art.cpp +5 -5
  12. package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +3 -0
  13. package/src/duckdb/src/execution/operator/helper/physical_vacuum.cpp +3 -0
  14. package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp +3 -0
  15. package/src/duckdb/src/execution/partitionable_hashtable.cpp +14 -2
  16. package/src/duckdb/src/execution/physical_plan/plan_comparison_join.cpp +3 -0
  17. package/src/duckdb/src/function/aggregate/distributive/bitstring_agg.cpp +22 -7
  18. package/src/duckdb/src/function/aggregate/distributive/first.cpp +1 -0
  19. package/src/duckdb/src/function/aggregate/holistic/approximate_quantile.cpp +5 -2
  20. package/src/duckdb/src/function/cast/cast_function_set.cpp +1 -1
  21. package/src/duckdb/src/function/cast/enum_casts.cpp +25 -3
  22. package/src/duckdb/src/function/cast/list_casts.cpp +17 -4
  23. package/src/duckdb/src/function/cast/map_cast.cpp +5 -2
  24. package/src/duckdb/src/function/cast/string_cast.cpp +36 -10
  25. package/src/duckdb/src/function/cast/struct_cast.cpp +23 -3
  26. package/src/duckdb/src/function/cast/union_casts.cpp +33 -7
  27. package/src/duckdb/src/function/table/checkpoint.cpp +5 -1
  28. package/src/duckdb/src/function/table/system/duckdb_constraints.cpp +2 -2
  29. package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
  30. package/src/duckdb/src/include/duckdb/catalog/dependency_list.hpp +3 -0
  31. package/src/duckdb/src/include/duckdb/common/optional_ptr.hpp +45 -0
  32. package/src/duckdb/src/include/duckdb/execution/aggregate_hashtable.hpp +1 -0
  33. package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +2 -2
  34. package/src/duckdb/src/include/duckdb/execution/partitionable_hashtable.hpp +3 -0
  35. package/src/duckdb/src/include/duckdb/function/cast/bound_cast_data.hpp +84 -0
  36. package/src/duckdb/src/include/duckdb/function/cast/cast_function_set.hpp +2 -2
  37. package/src/duckdb/src/include/duckdb/function/cast/default_casts.hpp +28 -64
  38. package/src/duckdb/src/include/duckdb/parser/transformer.hpp +3 -0
  39. package/src/duckdb/src/include/duckdb/storage/data_table.hpp +2 -2
  40. package/src/duckdb/src/include/duckdb/storage/index.hpp +4 -3
  41. package/src/duckdb/src/include/duckdb/storage/statistics/distinct_statistics.hpp +2 -0
  42. package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp +2 -2
  43. package/src/duckdb/src/optimizer/pushdown/pushdown_aggregate.cpp +33 -5
  44. package/src/duckdb/src/optimizer/rule/move_constants.cpp +8 -2
  45. package/src/duckdb/src/parser/transform/expression/transform_function.cpp +17 -1
  46. package/src/duckdb/src/parser/transform/statement/transform_select_node.cpp +1 -2
  47. package/src/duckdb/src/planner/binder/query_node/bind_select_node.cpp +25 -13
  48. package/src/duckdb/src/planner/binder/statement/bind_copy.cpp +5 -3
  49. package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +7 -0
  50. package/src/duckdb/src/planner/binder/statement/bind_insert.cpp +10 -6
  51. package/src/duckdb/src/planner/binder/statement/bind_update.cpp +3 -1
  52. package/src/duckdb/src/planner/expression_binder/having_binder.cpp +3 -0
  53. package/src/duckdb/src/storage/data_table.cpp +15 -13
  54. package/src/duckdb/src/storage/index.cpp +12 -1
  55. package/src/duckdb/src/storage/local_storage.cpp +20 -23
  56. package/src/duckdb/src/storage/statistics/column_statistics.cpp +1 -2
  57. package/src/duckdb/src/storage/statistics/distinct_statistics.cpp +4 -0
  58. package/src/duckdb/src/storage/table/row_group.cpp +6 -1
@@ -150,8 +150,8 @@ public:
150
150
 
151
151
  //! Append a chunk with the row ids [row_start, ..., row_start + chunk.size()] to all indexes of the table, returns
152
152
  //! whether or not the append succeeded
153
- bool AppendToIndexes(DataChunk &chunk, row_t row_start);
154
- static bool AppendToIndexes(TableIndexList &indexes, DataChunk &chunk, row_t row_start);
153
+ PreservedError AppendToIndexes(DataChunk &chunk, row_t row_start);
154
+ static PreservedError AppendToIndexes(TableIndexList &indexes, DataChunk &chunk, row_t row_start);
155
155
  //! Remove a chunk with the row ids [row_start, ..., row_start + chunk.size()] from all indexes of the table
156
156
  void RemoveFromIndexes(TableAppendState &state, DataChunk &chunk, row_t row_start);
157
157
  //! Remove the chunk with the specified set of row identifiers from all indexes of the table
@@ -80,9 +80,9 @@ public:
80
80
  //! Obtain a lock on the index
81
81
  virtual void InitializeLock(IndexLock &state);
82
82
  //! Called when data is appended to the index. The lock obtained from InitializeLock must be held
83
- virtual bool Append(IndexLock &state, DataChunk &entries, Vector &row_identifiers) = 0;
83
+ virtual PreservedError Append(IndexLock &state, DataChunk &entries, Vector &row_identifiers) = 0;
84
84
  //! Obtains a lock and calls Append while holding that lock
85
- bool Append(DataChunk &entries, Vector &row_identifiers);
85
+ PreservedError Append(DataChunk &entries, Vector &row_identifiers);
86
86
  //! Verify that data can be appended to the index without a constraint violation
87
87
  virtual void VerifyAppend(DataChunk &chunk) = 0;
88
88
  //! Verify that data can be appended to the index without a constraint violation using the conflict manager
@@ -96,7 +96,7 @@ public:
96
96
  void Delete(DataChunk &entries, Vector &row_identifiers);
97
97
 
98
98
  //! Insert a chunk of entries into the index
99
- virtual bool Insert(IndexLock &lock, DataChunk &input, Vector &row_identifiers) = 0;
99
+ virtual PreservedError Insert(IndexLock &lock, DataChunk &input, Vector &row_identifiers) = 0;
100
100
 
101
101
  //! Merge another index into this index. The lock obtained from InitializeLock must be held, and the other
102
102
  //! index must also be locked during the merge
@@ -147,6 +147,7 @@ public:
147
147
 
148
148
  //! Execute the index expressions on an input chunk
149
149
  void ExecuteExpressions(DataChunk &input, DataChunk &result);
150
+ static string AppendRowError(DataChunk &input, idx_t index);
150
151
 
151
152
  protected:
152
153
  //! Lock used for any changes to the index
@@ -46,6 +46,8 @@ public:
46
46
  string ToString() const;
47
47
  idx_t GetCount() const;
48
48
 
49
+ static bool TypeIsSupported(const LogicalType &type);
50
+
49
51
  private:
50
52
  //! For distinct statistics we sample the input to speed up insertions
51
53
  static constexpr const double SAMPLE_RATE = 0.1;
@@ -88,8 +88,8 @@ public:
88
88
 
89
89
  void AppendToIndexes(DuckTransaction &transaction, TableAppendState &append_state, idx_t append_count,
90
90
  bool append_to_table);
91
- bool AppendToIndexes(DuckTransaction &transaction, RowGroupCollection &source, TableIndexList &index_list,
92
- const vector<LogicalType> &table_types, row_t &start_row);
91
+ PreservedError AppendToIndexes(DuckTransaction &transaction, RowGroupCollection &source, TableIndexList &index_list,
92
+ const vector<LogicalType> &table_types, row_t &start_row);
93
93
 
94
94
  //! Creates an optimistic writer for this table
95
95
  OptimisticDataWriter *CreateOptimisticWriter();
@@ -9,6 +9,14 @@ namespace duckdb {
9
9
 
10
10
  using Filter = FilterPushdown::Filter;
11
11
 
12
+ static void ExtractFilterBindings(Expression &expr, vector<ColumnBinding> &bindings) {
13
+ if (expr.type == ExpressionType::BOUND_COLUMN_REF) {
14
+ auto &colref = (BoundColumnRefExpression &)expr;
15
+ bindings.push_back(colref.binding);
16
+ }
17
+ ExpressionIterator::EnumerateChildren(expr, [&](Expression &child) { ExtractFilterBindings(child, bindings); });
18
+ }
19
+
12
20
  static unique_ptr<Expression> ReplaceGroupBindings(LogicalAggregate &proj, unique_ptr<Expression> expr) {
13
21
  if (expr->type == ExpressionType::BOUND_COLUMN_REF) {
14
22
  auto &colref = (BoundColumnRefExpression &)*expr;
@@ -40,14 +48,34 @@ unique_ptr<LogicalOperator> FilterPushdown::PushdownAggregate(unique_ptr<Logical
40
48
  // filter on GROUPINGS function: cannot pushdown
41
49
  continue;
42
50
  }
43
- // if there are any empty grouping sets, we cannot push down filters
44
- bool has_empty_grouping_sets = false;
51
+ // no aggregate! we are filtering on a group
52
+ // we can only push this down if the filter is in all grouping sets
53
+ vector<ColumnBinding> bindings;
54
+ ExtractFilterBindings(*f.filter, bindings);
55
+
56
+ bool can_pushdown_filter = true;
57
+ if (aggr.grouping_sets.empty()) {
58
+ // empty grouping set - we cannot pushdown the filter
59
+ can_pushdown_filter = false;
60
+ }
45
61
  for (auto &grp : aggr.grouping_sets) {
46
- if (grp.empty()) {
47
- has_empty_grouping_sets = true;
62
+ // check for each of the grouping sets if they contain all groups
63
+ if (bindings.empty()) {
64
+ // we can never push down empty grouping sets
65
+ can_pushdown_filter = false;
66
+ break;
67
+ }
68
+ for (auto &binding : bindings) {
69
+ if (grp.find(binding.column_index) == grp.end()) {
70
+ can_pushdown_filter = false;
71
+ break;
72
+ }
73
+ }
74
+ if (!can_pushdown_filter) {
75
+ break;
48
76
  }
49
77
  }
50
- if (has_empty_grouping_sets) {
78
+ if (!can_pushdown_filter) {
51
79
  continue;
52
80
  }
53
81
  // no aggregate! we can push this down
@@ -73,7 +73,10 @@ unique_ptr<Expression> MoveConstantsRule::Apply(LogicalOperator &op, vector<Expr
73
73
  }
74
74
  auto result_value = Value::HUGEINT(outer_value);
75
75
  if (!result_value.DefaultTryCastAs(constant_type)) {
76
- // if the cast is not possible then the comparison is not possible
76
+ // if the cast is not possible then an equality comparison is not possible
77
+ if (comparison->type != ExpressionType::COMPARE_EQUAL) {
78
+ return nullptr;
79
+ }
77
80
  return ExpressionRewriter::ConstantOrNull(std::move(arithmetic->children[arithmetic_child_index]),
78
81
  Value::BOOLEAN(false));
79
82
  }
@@ -86,7 +89,10 @@ unique_ptr<Expression> MoveConstantsRule::Apply(LogicalOperator &op, vector<Expr
86
89
  }
87
90
  auto result_value = Value::HUGEINT(inner_value);
88
91
  if (!result_value.DefaultTryCastAs(constant_type)) {
89
- // if the cast is not possible then the comparison is not possible
92
+ // if the cast is not possible then an equality comparison is not possible
93
+ if (comparison->type != ExpressionType::COMPARE_EQUAL) {
94
+ return nullptr;
95
+ }
90
96
  return ExpressionRewriter::ConstantOrNull(std::move(arithmetic->children[arithmetic_child_index]),
91
97
  Value::BOOLEAN(false));
92
98
  }
@@ -105,6 +105,16 @@ bool Transformer::ExpressionIsEmptyStar(ParsedExpression &expr) {
105
105
  return false;
106
106
  }
107
107
 
108
+ bool Transformer::InWindowDefinition() {
109
+ if (in_window_definition) {
110
+ return true;
111
+ }
112
+ if (parent) {
113
+ return parent->InWindowDefinition();
114
+ }
115
+ return false;
116
+ }
117
+
108
118
  unique_ptr<ParsedExpression> Transformer::TransformFuncCall(duckdb_libpgquery::PGFuncCall *root) {
109
119
  auto name = root->funcname;
110
120
  string catalog, schema, function_name;
@@ -124,7 +134,7 @@ unique_ptr<ParsedExpression> Transformer::TransformFuncCall(duckdb_libpgquery::P
124
134
  schema = INVALID_SCHEMA;
125
135
  function_name = reinterpret_cast<duckdb_libpgquery::PGValue *>(name->head->data.ptr_value)->val.str;
126
136
  } else {
127
- throw InternalException("TransformFuncCall - Expected 1, 2 or 3 qualifications");
137
+ throw ParserException("TransformFuncCall - Expected 1, 2 or 3 qualifications");
128
138
  }
129
139
 
130
140
  // transform children
@@ -139,6 +149,10 @@ unique_ptr<ParsedExpression> Transformer::TransformFuncCall(duckdb_libpgquery::P
139
149
 
140
150
  auto lowercase_name = StringUtil::Lower(function_name);
141
151
  if (root->over) {
152
+ if (InWindowDefinition()) {
153
+ throw ParserException("window functions are not allowed in window definitions");
154
+ }
155
+
142
156
  const auto win_fun_type = WindowToExpressionType(lowercase_name);
143
157
  if (win_fun_type == ExpressionType::INVALID) {
144
158
  throw InternalException("Unknown/unsupported window function");
@@ -218,8 +232,10 @@ unique_ptr<ParsedExpression> Transformer::TransformFuncCall(duckdb_libpgquery::P
218
232
  window_ref = it->second;
219
233
  D_ASSERT(window_ref);
220
234
  }
235
+ in_window_definition = true;
221
236
  TransformWindowDef(window_ref, expr.get());
222
237
  TransformWindowFrame(window_spec, expr.get());
238
+ in_window_definition = false;
223
239
  expr->query_location = root->location;
224
240
  return std::move(expr);
225
241
  }
@@ -26,8 +26,7 @@ unique_ptr<QueryNode> Transformer::TransformSelectInternal(duckdb_libpgquery::PG
26
26
  auto window_def = reinterpret_cast<duckdb_libpgquery::PGWindowDef *>(window_ele->data.ptr_value);
27
27
  D_ASSERT(window_def);
28
28
  D_ASSERT(window_def->name);
29
- auto window_name = StringUtil::Lower(string(window_def->name));
30
-
29
+ string window_name(window_def->name);
31
30
  auto it = window_clauses.find(window_name);
32
31
  if (it != window_clauses.end()) {
33
32
  throw ParserException("window \"%s\" is already defined", window_name);
@@ -426,12 +426,16 @@ unique_ptr<BoundQueryNode> Binder::BindSelectNode(SelectNode &statement, unique_
426
426
  // after that, we bind to the SELECT list
427
427
  SelectBinder select_binder(*this, context, *result, info, alias_map);
428
428
  vector<LogicalType> internal_sql_types;
429
+ vector<idx_t> group_by_all_indexes;
429
430
  for (idx_t i = 0; i < statement.select_list.size(); i++) {
430
431
  bool is_window = statement.select_list[i]->IsWindow();
431
432
  idx_t unnest_count = result->unnests.size();
432
433
  LogicalType result_type;
433
434
  auto expr = select_binder.Bind(statement.select_list[i], &result_type);
434
- if (statement.aggregate_handling == AggregateHandling::FORCE_AGGREGATES && select_binder.HasBoundColumns()) {
435
+ bool is_original_column = i < result->column_count;
436
+ bool can_group_by_all =
437
+ statement.aggregate_handling == AggregateHandling::FORCE_AGGREGATES && is_original_column;
438
+ if (can_group_by_all && select_binder.HasBoundColumns()) {
435
439
  if (select_binder.BoundAggregates()) {
436
440
  throw BinderException("Cannot mix aggregates with non-aggregated columns!");
437
441
  }
@@ -443,20 +447,25 @@ unique_ptr<BoundQueryNode> Binder::BindSelectNode(SelectNode &statement, unique_
443
447
  }
444
448
  // we are forcing aggregates, and the node has columns bound
445
449
  // this entry becomes a group
446
- auto group_ref = make_unique<BoundColumnRefExpression>(
447
- expr->return_type, ColumnBinding(result->group_index, result->groups.group_expressions.size()));
448
- result->groups.group_expressions.push_back(std::move(expr));
449
- expr = std::move(group_ref);
450
+ group_by_all_indexes.push_back(i);
450
451
  }
451
452
  result->select_list.push_back(std::move(expr));
452
453
  if (i < result->column_count) {
453
454
  result->types.push_back(result_type);
454
455
  }
455
456
  internal_sql_types.push_back(result_type);
456
- if (statement.aggregate_handling == AggregateHandling::FORCE_AGGREGATES) {
457
+ if (can_group_by_all) {
457
458
  select_binder.ResetBindings();
458
459
  }
459
460
  }
461
+ // push the GROUP BY ALL expressions into the group set
462
+ for (auto &group_by_all_index : group_by_all_indexes) {
463
+ auto &expr = result->select_list[group_by_all_index];
464
+ auto group_ref = make_unique<BoundColumnRefExpression>(
465
+ expr->return_type, ColumnBinding(result->group_index, result->groups.group_expressions.size()));
466
+ result->groups.group_expressions.push_back(std::move(expr));
467
+ expr = std::move(group_ref);
468
+ }
460
469
  result->need_prune = result->select_list.size() > result->column_count;
461
470
 
462
471
  // in the normal select binder, we bind columns as if there is no aggregation
@@ -467,16 +476,19 @@ unique_ptr<BoundQueryNode> Binder::BindSelectNode(SelectNode &statement, unique_
467
476
  !result->groups.grouping_sets.empty()) {
468
477
  if (statement.aggregate_handling == AggregateHandling::NO_AGGREGATES_ALLOWED) {
469
478
  throw BinderException("Aggregates cannot be present in a Project relation!");
470
- } else if (statement.aggregate_handling == AggregateHandling::STANDARD_HANDLING) {
471
- if (select_binder.HasBoundColumns()) {
472
- auto &bound_columns = select_binder.GetBoundColumns();
473
- string error;
474
- error = "column \"%s\" must appear in the GROUP BY clause or must be part of an aggregate function.";
479
+ } else if (select_binder.HasBoundColumns()) {
480
+ auto &bound_columns = select_binder.GetBoundColumns();
481
+ string error;
482
+ error = "column \"%s\" must appear in the GROUP BY clause or must be part of an aggregate function.";
483
+ if (statement.aggregate_handling == AggregateHandling::FORCE_AGGREGATES) {
484
+ error += "\nGROUP BY ALL will only group entries in the SELECT list. Add it to the SELECT list or "
485
+ "GROUP BY this entry explicitly.";
486
+ } else {
475
487
  error += "\nEither add it to the GROUP BY list, or use \"ANY_VALUE(%s)\" if the exact value of \"%s\" "
476
488
  "is not important.";
477
- throw BinderException(FormatError(bound_columns[0].query_location, error, bound_columns[0].name,
478
- bound_columns[0].name, bound_columns[0].name));
479
489
  }
490
+ throw BinderException(FormatError(bound_columns[0].query_location, error, bound_columns[0].name,
491
+ bound_columns[0].name, bound_columns[0].name));
480
492
  }
481
493
  }
482
494
 
@@ -158,7 +158,9 @@ BoundStatement Binder::BindCopyFrom(CopyStatement &stmt) {
158
158
  result.types = {LogicalType::BIGINT};
159
159
  result.names = {"Count"};
160
160
 
161
- D_ASSERT(!stmt.info->table.empty());
161
+ if (stmt.info->table.empty()) {
162
+ throw ParserException("COPY FROM requires a table name to be specified");
163
+ }
162
164
  // COPY FROM a file
163
165
  // generate an insert statement for the the to-be-inserted table
164
166
  InsertStatement insert;
@@ -185,7 +187,7 @@ BoundStatement Binder::BindCopyFrom(CopyStatement &stmt) {
185
187
  vector<string> expected_names;
186
188
  if (!bound_insert.column_index_map.empty()) {
187
189
  expected_names.resize(bound_insert.expected_types.size());
188
- for (auto &col : table->GetColumns().Logical()) {
190
+ for (auto &col : table->GetColumns().Physical()) {
189
191
  auto i = col.Physical();
190
192
  if (bound_insert.column_index_map[i] != DConstants::INVALID_INDEX) {
191
193
  expected_names[bound_insert.column_index_map[i]] = col.Name();
@@ -193,7 +195,7 @@ BoundStatement Binder::BindCopyFrom(CopyStatement &stmt) {
193
195
  }
194
196
  } else {
195
197
  expected_names.reserve(bound_insert.expected_types.size());
196
- for (auto &col : table->GetColumns().Logical()) {
198
+ for (auto &col : table->GetColumns().Physical()) {
197
199
  expected_names.push_back(col.Name());
198
200
  }
199
201
  }
@@ -119,9 +119,15 @@ static void BindConstraints(Binder &binder, BoundCreateTableInfo &info) {
119
119
  fk.info.type == ForeignKeyType::FK_TYPE_SELF_REFERENCE_TABLE);
120
120
  physical_index_set_t fk_key_set, pk_key_set;
121
121
  for (idx_t i = 0; i < fk.info.pk_keys.size(); i++) {
122
+ if (pk_key_set.find(fk.info.pk_keys[i]) != pk_key_set.end()) {
123
+ throw BinderException("Duplicate primary key referenced in FOREIGN KEY constraint");
124
+ }
122
125
  pk_key_set.insert(fk.info.pk_keys[i]);
123
126
  }
124
127
  for (idx_t i = 0; i < fk.info.fk_keys.size(); i++) {
128
+ if (fk_key_set.find(fk.info.fk_keys[i]) != fk_key_set.end()) {
129
+ throw BinderException("Duplicate key specified in FOREIGN KEY constraint");
130
+ }
125
131
  fk_key_set.insert(fk.info.fk_keys[i]);
126
132
  }
127
133
  info.bound_constraints.push_back(
@@ -292,6 +298,7 @@ unique_ptr<BoundCreateTableInfo> Binder::BindCreateTableInfo(unique_ptr<CreateIn
292
298
  result->dependencies.AddDependency(type_dependency);
293
299
  }
294
300
  }
301
+ result->dependencies.VerifyDependencies(schema->catalog, result->Base().table);
295
302
  properties.allow_stream_result = false;
296
303
  return result;
297
304
  }
@@ -335,6 +335,16 @@ void Binder::BindOnConflictClause(LogicalInsert &insert, TableCatalogEntry &tabl
335
335
  ReplaceColumnBindings(*insert.on_conflict_condition, table_index, projection_index);
336
336
  }
337
337
 
338
+ if (insert.action_type == OnConflictAction::REPLACE) {
339
+ D_ASSERT(on_conflict.set_info == nullptr);
340
+ on_conflict.set_info = CreateSetInfoForReplace(table, stmt);
341
+ insert.action_type = OnConflictAction::UPDATE;
342
+ }
343
+ if (on_conflict.set_info && on_conflict.set_info->columns.empty()) {
344
+ // if we are doing INSERT OR REPLACE on a table with no columns outside of the primary key column
345
+ // convert to INSERT OR IGNORE
346
+ insert.action_type = OnConflictAction::NOTHING;
347
+ }
338
348
  if (insert.action_type == OnConflictAction::NOTHING) {
339
349
  if (!insert.on_conflict_condition) {
340
350
  return;
@@ -346,15 +356,9 @@ void Binder::BindOnConflictClause(LogicalInsert &insert, TableCatalogEntry &tabl
346
356
  insert.columns_to_fetch = table_binding->GetBoundColumnIds();
347
357
  return;
348
358
  }
349
- if (insert.action_type == OnConflictAction::REPLACE) {
350
- D_ASSERT(on_conflict.set_info == nullptr);
351
- on_conflict.set_info = CreateSetInfoForReplace(table, stmt);
352
- insert.action_type = OnConflictAction::UPDATE;
353
- }
354
359
 
355
360
  D_ASSERT(on_conflict.set_info);
356
361
  auto &set_info = *on_conflict.set_info;
357
- D_ASSERT(!set_info.columns.empty());
358
362
  D_ASSERT(set_info.columns.size() == set_info.expressions.size());
359
363
 
360
364
  if (set_info.condition) {
@@ -195,11 +195,13 @@ BoundStatement Binder::Bind(UpdateStatement &stmt) {
195
195
  AddCTEMap(stmt.cte_map);
196
196
 
197
197
  if (stmt.from_table) {
198
+ auto from_binder = Binder::CreateBinder(context, this);
198
199
  BoundJoinRef bound_crossproduct(JoinRefType::CROSS);
199
200
  bound_crossproduct.left = std::move(bound_table);
200
- bound_crossproduct.right = Bind(*stmt.from_table);
201
+ bound_crossproduct.right = from_binder->Bind(*stmt.from_table);
201
202
  root = CreatePlan(bound_crossproduct);
202
203
  get = (LogicalGet *)root->children[0].get();
204
+ bind_context.AddContext(std::move(from_binder->bind_context));
203
205
  } else {
204
206
  root = CreatePlan(*bound_table);
205
207
  get = (LogicalGet *)root.get();
@@ -19,6 +19,9 @@ BindResult HavingBinder::BindColumnRef(unique_ptr<ParsedExpression> *expr_ptr, i
19
19
  auto &expr = (ColumnRefExpression &)**expr_ptr;
20
20
  auto alias_result = column_alias_binder.BindAlias(*this, expr, depth, root_expression);
21
21
  if (!alias_result.HasError()) {
22
+ if (depth > 0) {
23
+ throw BinderException("Having clause cannot reference alias in correlated subquery");
24
+ }
22
25
  return alias_result;
23
26
  }
24
27
  if (aggregate_handling == AggregateHandling::FORCE_AGGREGATES) {
@@ -819,9 +819,10 @@ void DataTable::RevertAppend(idx_t start_row, idx_t count) {
819
819
  //===--------------------------------------------------------------------===//
820
820
  // Indexes
821
821
  //===--------------------------------------------------------------------===//
822
- bool DataTable::AppendToIndexes(TableIndexList &indexes, DataChunk &chunk, row_t row_start) {
822
+ PreservedError DataTable::AppendToIndexes(TableIndexList &indexes, DataChunk &chunk, row_t row_start) {
823
+ PreservedError error;
823
824
  if (indexes.Empty()) {
824
- return true;
825
+ return error;
825
826
  }
826
827
  // first generate the vector of row identifiers
827
828
  Vector row_identifiers(LogicalType::ROW_TYPE);
@@ -832,11 +833,13 @@ bool DataTable::AppendToIndexes(TableIndexList &indexes, DataChunk &chunk, row_t
832
833
  // now append the entries to the indices
833
834
  indexes.Scan([&](Index &index) {
834
835
  try {
835
- if (!index.Append(chunk, row_identifiers)) {
836
- append_failed = true;
837
- return true;
838
- }
839
- } catch (...) {
836
+ error = index.Append(chunk, row_identifiers);
837
+ } catch (Exception &ex) {
838
+ error = PreservedError(ex);
839
+ } catch (std::exception &ex) {
840
+ error = PreservedError(ex);
841
+ }
842
+ if (error) {
840
843
  append_failed = true;
841
844
  return true;
842
845
  }
@@ -850,12 +853,11 @@ bool DataTable::AppendToIndexes(TableIndexList &indexes, DataChunk &chunk, row_t
850
853
  for (auto *index : already_appended) {
851
854
  index->Delete(chunk, row_identifiers);
852
855
  }
853
- return false;
854
856
  }
855
- return true;
857
+ return error;
856
858
  }
857
859
 
858
- bool DataTable::AppendToIndexes(DataChunk &chunk, row_t row_start) {
860
+ PreservedError DataTable::AppendToIndexes(DataChunk &chunk, row_t row_start) {
859
861
  D_ASSERT(is_root);
860
862
  return AppendToIndexes(info->indexes, chunk, row_start);
861
863
  }
@@ -1204,9 +1206,9 @@ void DataTable::WALAddIndex(ClientContext &context, unique_ptr<Index> index,
1204
1206
  index->ExecuteExpressions(intermediate, result);
1205
1207
 
1206
1208
  // insert into the index
1207
- if (!index->Insert(lock, result, intermediate.data[intermediate.ColumnCount() - 1])) {
1208
- throw InternalException("Error during WAL replay. Can't create unique index, table contains "
1209
- "duplicate data on indexed column(s).");
1209
+ auto error = index->Insert(lock, result, intermediate.data[intermediate.ColumnCount() - 1]);
1210
+ if (error) {
1211
+ throw InternalException("Error during WAL replay: %s", error.Message());
1210
1212
  }
1211
1213
  }
1212
1214
  }
@@ -36,7 +36,7 @@ void Index::InitializeLock(IndexLock &state) {
36
36
  state.index_lock = unique_lock<mutex>(lock);
37
37
  }
38
38
 
39
- bool Index::Append(DataChunk &entries, Vector &row_identifiers) {
39
+ PreservedError Index::Append(DataChunk &entries, Vector &row_identifiers) {
40
40
  IndexLock state;
41
41
  InitializeLock(state);
42
42
  return Append(state, entries, row_identifiers);
@@ -90,4 +90,15 @@ BlockPointer Index::Serialize(MetaBlockWriter &writer) {
90
90
  throw NotImplementedException("The implementation of this index serialization does not exist.");
91
91
  }
92
92
 
93
+ string Index::AppendRowError(DataChunk &input, idx_t index) {
94
+ string error;
95
+ for (idx_t c = 0; c < input.ColumnCount(); c++) {
96
+ if (c > 0) {
97
+ error += ", ";
98
+ }
99
+ error += input.GetValue(c, index).ToString();
100
+ }
101
+ return error;
102
+ }
103
+
93
104
  } // namespace duckdb
@@ -197,16 +197,16 @@ void LocalTableStorage::FlushToDisk() {
197
197
  optimistic_writer.FinalFlush();
198
198
  }
199
199
 
200
- bool LocalTableStorage::AppendToIndexes(DuckTransaction &transaction, RowGroupCollection &source,
201
- TableIndexList &index_list, const vector<LogicalType> &table_types,
202
- row_t &start_row) {
200
+ PreservedError LocalTableStorage::AppendToIndexes(DuckTransaction &transaction, RowGroupCollection &source,
201
+ TableIndexList &index_list, const vector<LogicalType> &table_types,
202
+ row_t &start_row) {
203
203
  // only need to scan for index append
204
204
  // figure out which columns we need to scan for the set of indexes
205
205
  auto columns = index_list.GetRequiredColumns();
206
206
  // create an empty mock chunk that contains all the correct types for the table
207
207
  DataChunk mock_chunk;
208
208
  mock_chunk.InitializeEmpty(table_types);
209
- bool success = true;
209
+ PreservedError error;
210
210
  source.Scan(transaction, columns, [&](DataChunk &chunk) -> bool {
211
211
  // construct the mock chunk by referencing the required columns
212
212
  for (idx_t i = 0; i < columns.size(); i++) {
@@ -214,28 +214,28 @@ bool LocalTableStorage::AppendToIndexes(DuckTransaction &transaction, RowGroupCo
214
214
  }
215
215
  mock_chunk.SetCardinality(chunk);
216
216
  // append this chunk to the indexes of the table
217
- if (!DataTable::AppendToIndexes(index_list, mock_chunk, start_row)) {
218
- success = false;
217
+ error = DataTable::AppendToIndexes(index_list, mock_chunk, start_row);
218
+ if (error) {
219
219
  return false;
220
220
  }
221
221
  start_row += chunk.size();
222
222
  return true;
223
223
  });
224
- return success;
224
+ return error;
225
225
  }
226
226
 
227
227
  void LocalTableStorage::AppendToIndexes(DuckTransaction &transaction, TableAppendState &append_state,
228
228
  idx_t append_count, bool append_to_table) {
229
- bool constraint_violated = false;
230
229
  if (append_to_table) {
231
230
  table->InitializeAppend(transaction, append_state, append_count);
232
231
  }
232
+ PreservedError error;
233
233
  if (append_to_table) {
234
234
  // appending: need to scan entire
235
235
  row_groups->Scan(transaction, [&](DataChunk &chunk) -> bool {
236
236
  // append this chunk to the indexes of the table
237
- if (!table->AppendToIndexes(chunk, append_state.current_row)) {
238
- constraint_violated = true;
237
+ error = table->AppendToIndexes(chunk, append_state.current_row);
238
+ if (error) {
239
239
  return false;
240
240
  }
241
241
  // append to base table
@@ -243,11 +243,10 @@ void LocalTableStorage::AppendToIndexes(DuckTransaction &transaction, TableAppen
243
243
  return true;
244
244
  });
245
245
  } else {
246
- constraint_violated = !AppendToIndexes(transaction, *row_groups, table->info->indexes, table->GetTypes(),
247
- append_state.current_row);
246
+ error = AppendToIndexes(transaction, *row_groups, table->info->indexes, table->GetTypes(),
247
+ append_state.current_row);
248
248
  }
249
- if (constraint_violated) {
250
- PreservedError error;
249
+ if (error) {
251
250
  // need to revert the append
252
251
  row_t current_row = append_state.row_start;
253
252
  // remove the data from the indexes, if there are any indexes
@@ -273,10 +272,7 @@ void LocalTableStorage::AppendToIndexes(DuckTransaction &transaction, TableAppen
273
272
  if (append_to_table) {
274
273
  table->RevertAppendInternal(append_state.row_start, append_count);
275
274
  }
276
- if (error) {
277
- error.Throw();
278
- }
279
- throw ConstraintException("PRIMARY KEY or UNIQUE constraint violated: duplicated key");
275
+ error.Throw();
280
276
  }
281
277
  }
282
278
 
@@ -412,8 +408,9 @@ void LocalStorage::Append(LocalAppendState &state, DataChunk &chunk) {
412
408
  // append to unique indices (if any)
413
409
  auto storage = state.storage;
414
410
  idx_t base_id = MAX_ROW_ID + storage->row_groups->GetTotalRows() + state.append_state.total_append_count;
415
- if (!DataTable::AppendToIndexes(storage->indexes, chunk, base_id)) {
416
- throw ConstraintException("PRIMARY KEY or UNIQUE constraint violated: duplicated key");
411
+ auto error = DataTable::AppendToIndexes(storage->indexes, chunk, base_id);
412
+ if (error) {
413
+ error.Throw();
417
414
  }
418
415
 
419
416
  //! Append the chunk to the local storage
@@ -434,9 +431,9 @@ void LocalStorage::LocalMerge(DataTable *table, RowGroupCollection &collection)
434
431
  if (!storage->indexes.Empty()) {
435
432
  // append data to indexes if required
436
433
  row_t base_id = MAX_ROW_ID + storage->row_groups->GetTotalRows();
437
- bool success = storage->AppendToIndexes(transaction, collection, storage->indexes, table->GetTypes(), base_id);
438
- if (!success) {
439
- throw ConstraintException("PRIMARY KEY or UNIQUE constraint violated: duplicated key");
434
+ auto error = storage->AppendToIndexes(transaction, collection, storage->indexes, table->GetTypes(), base_id);
435
+ if (error) {
436
+ error.Throw();
440
437
  }
441
438
  }
442
439
  storage->row_groups->MergeStorage(collection);
@@ -4,8 +4,7 @@
4
4
  namespace duckdb {
5
5
 
6
6
  ColumnStatistics::ColumnStatistics(BaseStatistics stats_p) : stats(std::move(stats_p)) {
7
- auto type = stats.GetType().InternalType();
8
- if (type != PhysicalType::LIST && type != PhysicalType::STRUCT) {
7
+ if (DistinctStatistics::TypeIsSupported(stats.GetType())) {
9
8
  distinct_stats = make_unique<DistinctStatistics>();
10
9
  }
11
10
  }
@@ -94,4 +94,8 @@ idx_t DistinctStatistics::GetCount() const {
94
94
  return MinValue<idx_t>(estimate, total_count);
95
95
  }
96
96
 
97
+ bool DistinctStatistics::TypeIsSupported(const LogicalType &type) {
98
+ return type.InternalType() != PhysicalType::LIST && type.InternalType() != PhysicalType::STRUCT;
99
+ }
100
+
97
101
  } // namespace duckdb
@@ -156,7 +156,11 @@ unique_ptr<RowGroup> RowGroup::AlterType(const LogicalType &target_type, idx_t c
156
156
  // scan the original table, and fill the new column with the transformed value
157
157
  InitializeScan(scan_state);
158
158
 
159
- Vector append_vector(target_type);
159
+ DataChunk append_chunk;
160
+ vector<LogicalType> append_types;
161
+ append_types.push_back(target_type);
162
+ append_chunk.Initialize(Allocator::DefaultAllocator(), append_types);
163
+ auto &append_vector = append_chunk.data[0];
160
164
  SegmentStatistics altered_col_stats(target_type);
161
165
  while (true) {
162
166
  // scan the table
@@ -166,6 +170,7 @@ unique_ptr<RowGroup> RowGroup::AlterType(const LogicalType &target_type, idx_t c
166
170
  break;
167
171
  }
168
172
  // execute the expression
173
+ append_chunk.Reset();
169
174
  executor.ExecuteExpression(scan_chunk, append_vector);
170
175
  column_data->Append(altered_col_stats.statistics, append_state, append_vector, scan_chunk.size());
171
176
  }