duckdb 0.8.2-dev2399.0 → 0.8.2-dev2669.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/binding.gyp +1 -0
  2. package/package.json +1 -1
  3. package/src/duckdb/extension/icu/icu-datepart.cpp +3 -3
  4. package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +1 -1
  5. package/src/duckdb/src/catalog/default/default_functions.cpp +5 -0
  6. package/src/duckdb/src/common/enum_util.cpp +35 -1
  7. package/src/duckdb/src/common/http_state.cpp +78 -0
  8. package/src/duckdb/src/core_functions/function_list.cpp +2 -2
  9. package/src/duckdb/src/core_functions/scalar/list/array_slice.cpp +314 -82
  10. package/src/duckdb/src/execution/expression_executor/execute_parameter.cpp +2 -2
  11. package/src/duckdb/src/execution/index/art/art.cpp +43 -31
  12. package/src/duckdb/src/execution/index/art/leaf.cpp +47 -33
  13. package/src/duckdb/src/execution/index/art/node.cpp +31 -24
  14. package/src/duckdb/src/execution/index/art/prefix.cpp +100 -16
  15. package/src/duckdb/src/execution/operator/schema/physical_create_index.cpp +54 -31
  16. package/src/duckdb/src/execution/physical_plan/plan_create_index.cpp +32 -15
  17. package/src/duckdb/src/function/table/arrow/arrow_duck_schema.cpp +57 -0
  18. package/src/duckdb/src/function/table/arrow.cpp +95 -92
  19. package/src/duckdb/src/function/table/arrow_conversion.cpp +45 -68
  20. package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
  21. package/src/duckdb/src/include/duckdb/common/case_insensitive_map.hpp +1 -0
  22. package/src/duckdb/src/include/duckdb/common/enum_util.hpp +8 -0
  23. package/src/duckdb/src/include/duckdb/common/helper.hpp +8 -3
  24. package/src/duckdb/src/include/duckdb/common/http_state.hpp +61 -28
  25. package/src/duckdb/src/include/duckdb/common/types/value.hpp +4 -1
  26. package/src/duckdb/src/include/duckdb/core_functions/scalar/list_functions.hpp +4 -4
  27. package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +7 -5
  28. package/src/duckdb/src/include/duckdb/execution/index/art/leaf.hpp +6 -6
  29. package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp +6 -0
  30. package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp +9 -11
  31. package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_create_index.hpp +8 -1
  32. package/src/duckdb/src/include/duckdb/function/table/arrow/arrow_duck_schema.hpp +99 -0
  33. package/src/duckdb/src/include/duckdb/function/table/arrow.hpp +6 -36
  34. package/src/duckdb/src/include/duckdb/main/capi/capi_internal.hpp +3 -1
  35. package/src/duckdb/src/include/duckdb/main/client_context.hpp +15 -14
  36. package/src/duckdb/src/include/duckdb/main/prepared_statement.hpp +73 -5
  37. package/src/duckdb/src/include/duckdb/main/prepared_statement_data.hpp +6 -6
  38. package/src/duckdb/src/include/duckdb/parser/expression/operator_expression.hpp +20 -3
  39. package/src/duckdb/src/include/duckdb/parser/expression/parameter_expression.hpp +17 -1
  40. package/src/duckdb/src/include/duckdb/parser/statement/execute_statement.hpp +1 -1
  41. package/src/duckdb/src/include/duckdb/parser/transformer.hpp +5 -3
  42. package/src/duckdb/src/include/duckdb/planner/bound_parameter_map.hpp +2 -1
  43. package/src/duckdb/src/include/duckdb/planner/expression/bound_parameter_data.hpp +20 -5
  44. package/src/duckdb/src/include/duckdb/planner/expression/bound_parameter_expression.hpp +3 -3
  45. package/src/duckdb/src/include/duckdb/planner/planner.hpp +4 -3
  46. package/src/duckdb/src/include/duckdb/storage/object_cache.hpp +1 -1
  47. package/src/duckdb/src/include/duckdb/verification/prepared_statement_verifier.hpp +1 -1
  48. package/src/duckdb/src/include/duckdb.h +16 -0
  49. package/src/duckdb/src/main/capi/pending-c.cpp +6 -0
  50. package/src/duckdb/src/main/capi/prepared-c.cpp +52 -4
  51. package/src/duckdb/src/main/client_context.cpp +27 -17
  52. package/src/duckdb/src/main/client_verify.cpp +17 -0
  53. package/src/duckdb/src/main/extension/extension_helper.cpp +2 -1
  54. package/src/duckdb/src/main/prepared_statement.cpp +38 -11
  55. package/src/duckdb/src/main/prepared_statement_data.cpp +23 -18
  56. package/src/duckdb/src/parser/expression/parameter_expression.cpp +7 -7
  57. package/src/duckdb/src/parser/statement/execute_statement.cpp +2 -2
  58. package/src/duckdb/src/parser/transform/expression/transform_array_access.cpp +13 -4
  59. package/src/duckdb/src/parser/transform/expression/transform_param_ref.cpp +45 -26
  60. package/src/duckdb/src/parser/transform/statement/transform_prepare.cpp +28 -6
  61. package/src/duckdb/src/parser/transformer.cpp +27 -9
  62. package/src/duckdb/src/planner/binder/expression/bind_parameter_expression.cpp +10 -10
  63. package/src/duckdb/src/planner/binder/statement/bind_execute.cpp +13 -7
  64. package/src/duckdb/src/planner/expression/bound_parameter_expression.cpp +13 -13
  65. package/src/duckdb/src/planner/planner.cpp +7 -6
  66. package/src/duckdb/src/storage/checkpoint_manager.cpp +1 -1
  67. package/src/duckdb/src/storage/serialization/serialize_expression.cpp +3 -3
  68. package/src/duckdb/src/storage/serialization/serialize_parsed_expression.cpp +2 -2
  69. package/src/duckdb/src/verification/prepared_statement_verifier.cpp +16 -11
  70. package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp +1 -0
  71. package/src/duckdb/third_party/libpg_query/src_backend_parser_gram.cpp +12855 -12282
  72. package/src/duckdb/ub_src_common.cpp +2 -0
  73. package/src/duckdb/ub_src_function_table_arrow.cpp +2 -0
@@ -7,34 +7,34 @@
7
7
  namespace duckdb {
8
8
 
9
9
  BindResult ExpressionBinder::BindExpression(ParameterExpression &expr, idx_t depth) {
10
- D_ASSERT(expr.parameter_nr > 0);
11
- auto bound_parameter = make_uniq<BoundParameterExpression>(expr.parameter_nr);
10
+ auto bound_parameter = make_uniq<BoundParameterExpression>(expr.identifier);
12
11
  bound_parameter->alias = expr.alias;
13
12
  if (!binder.parameters) {
14
13
  throw BinderException("Unexpected prepared parameter. This type of statement can't be prepared!");
15
14
  }
16
- auto parameter_idx = expr.parameter_nr;
15
+ auto parameter_id = expr.identifier;
17
16
  // check if a parameter value has already been supplied
18
- if (parameter_idx <= binder.parameters->parameter_data.size()) {
17
+ if (binder.parameters->parameter_data.count(parameter_id)) {
19
18
  // it has! emit a constant directly
20
- auto &data = binder.parameters->parameter_data[parameter_idx - 1];
21
- auto constant = make_uniq<BoundConstantExpression>(data.value);
19
+ auto &data = binder.parameters->parameter_data[parameter_id];
20
+ auto constant = make_uniq<BoundConstantExpression>(data.GetValue());
22
21
  constant->alias = expr.alias;
23
22
  return BindResult(std::move(constant));
24
23
  }
25
- auto entry = binder.parameters->parameters.find(parameter_idx);
24
+
25
+ auto entry = binder.parameters->parameters.find(parameter_id);
26
26
  if (entry == binder.parameters->parameters.end()) {
27
27
  // no entry yet: create a new one
28
28
  auto data = make_shared<BoundParameterData>();
29
- data->return_type = binder.parameters->GetReturnType(parameter_idx - 1);
29
+ data->return_type = binder.parameters->GetReturnType(parameter_id);
30
30
  bound_parameter->return_type = data->return_type;
31
31
  bound_parameter->parameter_data = data;
32
- binder.parameters->parameters[parameter_idx] = std::move(data);
32
+ binder.parameters->parameters[parameter_id] = std::move(data);
33
33
  } else {
34
34
  // a prepared statement with this parameter index was already there: use it
35
35
  auto &data = entry->second;
36
36
  bound_parameter->parameter_data = data;
37
- bound_parameter->return_type = binder.parameters->GetReturnType(parameter_idx - 1);
37
+ bound_parameter->return_type = binder.parameters->GetReturnType(parameter_id);
38
38
  }
39
39
  return BindResult(std::move(bound_parameter));
40
40
  }
@@ -6,6 +6,7 @@
6
6
  #include "duckdb/main/client_config.hpp"
7
7
  #include "duckdb/main/client_data.hpp"
8
8
  #include "duckdb/execution/expression_executor.hpp"
9
+ #include "duckdb/common/case_insensitive_map.hpp"
9
10
 
10
11
  namespace duckdb {
11
12
 
@@ -23,24 +24,29 @@ BoundStatement Binder::Bind(ExecuteStatement &stmt) {
23
24
  // check if we need to rebind the prepared statement
24
25
  // this happens if the catalog changes, since in this case e.g. tables we relied on may have been deleted
25
26
  auto prepared = entry->second;
27
+ auto &named_param_map = prepared->unbound_statement->named_param_map;
26
28
 
29
+ PreparedStatement::VerifyParameters(stmt.named_values, named_param_map);
30
+
31
+ auto &mapped_named_values = stmt.named_values;
27
32
  // bind any supplied parameters
28
- vector<Value> bind_values;
33
+ case_insensitive_map_t<Value> bind_values;
29
34
  auto constant_binder = Binder::CreateBinder(context);
30
35
  constant_binder->SetCanContainNulls(true);
31
- for (idx_t i = 0; i < stmt.values.size(); i++) {
36
+ for (auto &pair : mapped_named_values) {
32
37
  ConstantBinder cbinder(*constant_binder, context, "EXECUTE statement");
33
- auto bound_expr = cbinder.Bind(stmt.values[i]);
38
+ auto bound_expr = cbinder.Bind(pair.second);
34
39
 
35
40
  Value value = ExpressionExecutor::EvaluateScalar(context, *bound_expr, true);
36
- bind_values.push_back(std::move(value));
41
+ bind_values[pair.first] = std::move(value);
37
42
  }
38
43
  unique_ptr<LogicalOperator> rebound_plan;
39
- if (prepared->RequireRebind(context, bind_values)) {
44
+
45
+ if (prepared->RequireRebind(context, &bind_values)) {
40
46
  // catalog was modified or statement does not have clear types: rebind the statement before running the execute
41
47
  Planner prepared_planner(context);
42
- for (idx_t i = 0; i < bind_values.size(); i++) {
43
- prepared_planner.parameter_data.emplace_back(bind_values[i]);
48
+ for (auto &pair : bind_values) {
49
+ prepared_planner.parameter_data.emplace(pair);
44
50
  }
45
51
  prepared = prepared_planner.PrepareSQLStatement(entry->second->unbound_statement->Copy());
46
52
  rebound_plan = std::move(prepared_planner.plan);
@@ -6,22 +6,22 @@
6
6
 
7
7
  namespace duckdb {
8
8
 
9
- BoundParameterExpression::BoundParameterExpression(idx_t parameter_nr)
9
+ BoundParameterExpression::BoundParameterExpression(const string &identifier)
10
10
  : Expression(ExpressionType::VALUE_PARAMETER, ExpressionClass::BOUND_PARAMETER,
11
11
  LogicalType(LogicalTypeId::UNKNOWN)),
12
- parameter_nr(parameter_nr) {
12
+ identifier(identifier) {
13
13
  }
14
14
 
15
- BoundParameterExpression::BoundParameterExpression(bound_parameter_map_t &global_parameter_set, idx_t parameter_nr,
15
+ BoundParameterExpression::BoundParameterExpression(bound_parameter_map_t &global_parameter_set, string identifier,
16
16
  LogicalType return_type,
17
17
  shared_ptr<BoundParameterData> parameter_data)
18
18
  : Expression(ExpressionType::VALUE_PARAMETER, ExpressionClass::BOUND_PARAMETER, std::move(return_type)),
19
- parameter_nr(parameter_nr) {
19
+ identifier(std::move(identifier)) {
20
20
  // check if we have already deserialized a parameter with this number
21
- auto entry = global_parameter_set.find(parameter_nr);
21
+ auto entry = global_parameter_set.find(this->identifier);
22
22
  if (entry == global_parameter_set.end()) {
23
23
  // we have not - store the entry we deserialized from this parameter expression
24
- global_parameter_set[parameter_nr] = parameter_data;
24
+ global_parameter_set[this->identifier] = parameter_data;
25
25
  } else {
26
26
  // we have! use the previously deserialized entry
27
27
  parameter_data = entry->second;
@@ -57,7 +57,7 @@ bool BoundParameterExpression::IsFoldable() const {
57
57
  }
58
58
 
59
59
  string BoundParameterExpression::ToString() const {
60
- return "$" + to_string(parameter_nr);
60
+ return "$" + identifier;
61
61
  }
62
62
 
63
63
  bool BoundParameterExpression::Equals(const BaseExpression &other_p) const {
@@ -65,17 +65,17 @@ bool BoundParameterExpression::Equals(const BaseExpression &other_p) const {
65
65
  return false;
66
66
  }
67
67
  auto &other = other_p.Cast<BoundParameterExpression>();
68
- return parameter_nr == other.parameter_nr;
68
+ return StringUtil::CIEquals(identifier, other.identifier);
69
69
  }
70
70
 
71
71
  hash_t BoundParameterExpression::Hash() const {
72
72
  hash_t result = Expression::Hash();
73
- result = CombineHash(duckdb::Hash(parameter_nr), result);
73
+ result = CombineHash(duckdb::Hash(identifier.c_str(), identifier.size()), result);
74
74
  return result;
75
75
  }
76
76
 
77
77
  unique_ptr<Expression> BoundParameterExpression::Copy() {
78
- auto result = make_uniq<BoundParameterExpression>(parameter_nr);
78
+ auto result = make_uniq<BoundParameterExpression>(identifier);
79
79
  result->parameter_data = parameter_data;
80
80
  result->return_type = return_type;
81
81
  result->CopyProperties(*this);
@@ -83,7 +83,7 @@ unique_ptr<Expression> BoundParameterExpression::Copy() {
83
83
  }
84
84
 
85
85
  void BoundParameterExpression::Serialize(FieldWriter &writer) const {
86
- writer.WriteField(parameter_nr);
86
+ writer.WriteString(identifier);
87
87
  writer.WriteSerializable(return_type);
88
88
  writer.WriteSerializable(*parameter_data);
89
89
  }
@@ -91,11 +91,11 @@ void BoundParameterExpression::Serialize(FieldWriter &writer) const {
91
91
  unique_ptr<Expression> BoundParameterExpression::Deserialize(ExpressionDeserializationState &state,
92
92
  FieldReader &reader) {
93
93
  auto &global_parameter_set = state.gstate.parameter_data;
94
- auto parameter_nr = reader.ReadRequired<idx_t>();
94
+ auto identifier = reader.ReadRequired<string>();
95
95
  auto return_type = reader.ReadRequiredSerializable<LogicalType, LogicalType>();
96
96
  auto parameter_data = reader.ReadRequiredSerializable<BoundParameterData, shared_ptr<BoundParameterData>>();
97
97
  auto result = unique_ptr<BoundParameterExpression>(new BoundParameterExpression(
98
- global_parameter_set, parameter_nr, std::move(return_type), std::move(parameter_data)));
98
+ global_parameter_set, std::move(identifier), std::move(return_type), std::move(parameter_data)));
99
99
  return std::move(result);
100
100
  }
101
101
 
@@ -83,15 +83,15 @@ void Planner::CreatePlan(SQLStatement &statement) {
83
83
 
84
84
  // set up a map of parameter number -> value entries
85
85
  for (auto &kv : bound_parameters.parameters) {
86
- auto parameter_index = kv.first;
87
- auto &parameter_data = kv.second;
86
+ auto &identifier = kv.first;
87
+ auto &param = kv.second;
88
88
  // check if the type of the parameter could be resolved
89
- if (!parameter_data->return_type.IsValid()) {
89
+ if (!param->return_type.IsValid()) {
90
90
  properties.bound_all_parameters = false;
91
91
  continue;
92
92
  }
93
- parameter_data->value = Value(parameter_data->return_type);
94
- value_map[parameter_index] = parameter_data;
93
+ param->SetValue(Value(param->return_type));
94
+ value_map[identifier] = param;
95
95
  }
96
96
  }
97
97
 
@@ -153,7 +153,8 @@ static bool OperatorSupportsSerialization(LogicalOperator &op) {
153
153
  return op.SupportSerialization();
154
154
  }
155
155
 
156
- void Planner::VerifyPlan(ClientContext &context, unique_ptr<LogicalOperator> &op, bound_parameter_map_t *map) {
156
+ void Planner::VerifyPlan(ClientContext &context, unique_ptr<LogicalOperator> &op,
157
+ optional_ptr<bound_parameter_map_t> map) {
157
158
  #ifdef DUCKDB_ALTERNATIVE_VERIFY
158
159
  // if alternate verification is enabled we run the original operator
159
160
  return;
@@ -394,7 +394,7 @@ void CheckpointReader::ReadIndex(ClientContext &context, MetaBlockReader &reader
394
394
  case IndexType::ART: {
395
395
  auto &storage = table_catalog.GetStorage();
396
396
  auto art = make_uniq<ART>(index_info.column_ids, TableIOManager::Get(storage), std::move(unbound_expressions),
397
- index_info.constraint_type, storage.db, root_block_id, root_offset);
397
+ index_info.constraint_type, storage.db, nullptr, root_block_id, root_offset);
398
398
  index_catalog.index = art.get();
399
399
  storage.info->indexes.AddIndex(std::move(art));
400
400
  break;
@@ -241,16 +241,16 @@ unique_ptr<Expression> BoundOperatorExpression::FormatDeserialize(FormatDeserial
241
241
 
242
242
  void BoundParameterExpression::FormatSerialize(FormatSerializer &serializer) const {
243
243
  Expression::FormatSerialize(serializer);
244
- serializer.WriteProperty("parameter_nr", parameter_nr);
244
+ serializer.WriteProperty("identifier", identifier);
245
245
  serializer.WriteProperty("return_type", return_type);
246
246
  serializer.WriteProperty("parameter_data", *parameter_data);
247
247
  }
248
248
 
249
249
  unique_ptr<Expression> BoundParameterExpression::FormatDeserialize(FormatDeserializer &deserializer) {
250
- auto parameter_nr = deserializer.ReadProperty<idx_t>("parameter_nr");
250
+ auto identifier = deserializer.ReadProperty<string>("identifier");
251
251
  auto return_type = deserializer.ReadProperty<LogicalType>("return_type");
252
252
  auto parameter_data = deserializer.ReadProperty<shared_ptr<BoundParameterData>>("parameter_data");
253
- auto result = duckdb::unique_ptr<BoundParameterExpression>(new BoundParameterExpression(deserializer.Get<bound_parameter_map_t &>(), parameter_nr, std::move(return_type), std::move(parameter_data)));
253
+ auto result = duckdb::unique_ptr<BoundParameterExpression>(new BoundParameterExpression(deserializer.Get<bound_parameter_map_t &>(), std::move(identifier), std::move(return_type), std::move(parameter_data)));
254
254
  return std::move(result);
255
255
  }
256
256
 
@@ -246,12 +246,12 @@ unique_ptr<ParsedExpression> OperatorExpression::FormatDeserialize(FormatDeseria
246
246
 
247
247
  void ParameterExpression::FormatSerialize(FormatSerializer &serializer) const {
248
248
  ParsedExpression::FormatSerialize(serializer);
249
- serializer.WriteProperty("parameter_nr", parameter_nr);
249
+ serializer.WriteProperty("identifier", identifier);
250
250
  }
251
251
 
252
252
  unique_ptr<ParsedExpression> ParameterExpression::FormatDeserialize(FormatDeserializer &deserializer) {
253
253
  auto result = duckdb::unique_ptr<ParameterExpression>(new ParameterExpression());
254
- deserializer.ReadProperty("parameter_nr", result->parameter_nr);
254
+ deserializer.ReadProperty("identifier", result->identifier);
255
255
  return std::move(result);
256
256
  }
257
257
 
@@ -23,6 +23,9 @@ void PreparedStatementVerifier::Extract() {
23
23
  ParsedExpressionIterator::EnumerateQueryNodeChildren(
24
24
  *select.node, [&](unique_ptr<ParsedExpression> &child) { ConvertConstants(child); });
25
25
  statement->n_param = values.size();
26
+ for (auto &kv : values) {
27
+ statement->named_param_map[kv.first] = 0;
28
+ }
26
29
  // create the PREPARE and EXECUTE statements
27
30
  string name = "__duckdb_verification_prepared_statement";
28
31
  auto prepare = make_uniq<PrepareStatement>();
@@ -31,7 +34,7 @@ void PreparedStatementVerifier::Extract() {
31
34
 
32
35
  auto execute = make_uniq<ExecuteStatement>();
33
36
  execute->name = name;
34
- execute->values = std::move(values);
37
+ execute->named_values = std::move(values);
35
38
 
36
39
  auto dealloc = make_uniq<DropStatement>();
37
40
  dealloc->info->type = CatalogType::PREPARED_STATEMENT;
@@ -49,19 +52,21 @@ void PreparedStatementVerifier::ConvertConstants(unique_ptr<ParsedExpression> &c
49
52
  child->alias = string();
50
53
  // check if the value already exists
51
54
  idx_t index = values.size();
52
- for (idx_t v_idx = 0; v_idx < values.size(); v_idx++) {
53
- if (values[v_idx]->Equals(*child)) {
54
- // duplicate value! refer to the original value
55
- index = v_idx;
56
- break;
57
- }
58
- }
59
- if (index == values.size()) {
60
- values.push_back(std::move(child));
55
+ auto identifier = std::to_string(index + 1);
56
+ const auto predicate = [&](const std::pair<const string, unique_ptr<ParsedExpression>> &pair) {
57
+ return pair.second->Equals(*child.get());
58
+ };
59
+ auto result = std::find_if(values.begin(), values.end(), predicate);
60
+ if (result == values.end()) {
61
+ // If it doesn't exist yet, add it
62
+ values[identifier] = std::move(child);
63
+ } else {
64
+ identifier = result->first;
61
65
  }
66
+
62
67
  // replace it with an expression
63
68
  auto parameter = make_uniq<ParameterExpression>();
64
- parameter->parameter_nr = index + 1;
69
+ parameter->identifier = identifier;
65
70
  parameter->alias = alias;
66
71
  child = std::move(parameter);
67
72
  return;
@@ -325,6 +325,7 @@ typedef struct PGAIndices {
325
325
  bool is_slice; /* true if slice (i.e., colon present) */
326
326
  PGNode *lidx; /* slice lower bound, if any */
327
327
  PGNode *uidx; /* subscript, or slice upper bound if any */
328
+ PGNode *step; /* slice step, if any */
328
329
  } PGAIndices;
329
330
 
330
331
  /*