duckdb 0.8.2-dev2133.0 → 0.8.2-dev2278.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. package/package.json +1 -1
  2. package/src/duckdb/extension/json/json_functions.cpp +2 -1
  3. package/src/duckdb/extension/parquet/parquet_extension.cpp +2 -1
  4. package/src/duckdb/src/catalog/catalog.cpp +2 -6
  5. package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +1 -4
  6. package/src/duckdb/src/catalog/catalog_entry/table_catalog_entry.cpp +0 -4
  7. package/src/duckdb/src/catalog/catalog_entry/type_catalog_entry.cpp +7 -26
  8. package/src/duckdb/src/catalog/catalog_set.cpp +0 -63
  9. package/src/duckdb/src/catalog/dependency_manager.cpp +0 -36
  10. package/src/duckdb/src/common/adbc/adbc.cpp +1 -1
  11. package/src/duckdb/src/common/extra_type_info.cpp +24 -46
  12. package/src/duckdb/src/common/field_writer.cpp +0 -1
  13. package/src/duckdb/src/common/file_system.cpp +6 -6
  14. package/src/duckdb/src/common/filename_pattern.cpp +1 -1
  15. package/src/duckdb/src/common/gzip_file_system.cpp +7 -12
  16. package/src/duckdb/src/common/multi_file_reader.cpp +8 -5
  17. package/src/duckdb/src/common/serializer/buffered_file_reader.cpp +0 -9
  18. package/src/duckdb/src/common/types.cpp +12 -56
  19. package/src/duckdb/src/common/virtual_file_system.cpp +4 -0
  20. package/src/duckdb/src/core_functions/scalar/union/union_tag.cpp +1 -1
  21. package/src/duckdb/src/execution/column_binding_resolver.cpp +3 -7
  22. package/src/duckdb/src/execution/operator/persistent/physical_copy_to_file.cpp +1 -1
  23. package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp +0 -10
  24. package/src/duckdb/src/execution/operator/schema/physical_attach.cpp +2 -1
  25. package/src/duckdb/src/execution/operator/schema/physical_create_type.cpp +2 -6
  26. package/src/duckdb/src/execution/physical_plan/plan_asof_join.cpp +1 -2
  27. package/src/duckdb/src/execution/physical_plan/plan_comparison_join.cpp +14 -1
  28. package/src/duckdb/src/execution/physical_plan/plan_create_index.cpp +3 -17
  29. package/src/duckdb/src/execution/physical_plan/plan_delim_join.cpp +2 -5
  30. package/src/duckdb/src/execution/physical_plan_generator.cpp +1 -5
  31. package/src/duckdb/src/function/cast/string_cast.cpp +0 -1
  32. package/src/duckdb/src/function/scalar_function.cpp +3 -19
  33. package/src/duckdb/src/function/table/arrow.cpp +1 -1
  34. package/src/duckdb/src/function/table/arrow_conversion.cpp +12 -10
  35. package/src/duckdb/src/function/table/read_csv.cpp +2 -1
  36. package/src/duckdb/src/function/table/system/test_all_types.cpp +4 -4
  37. package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
  38. package/src/duckdb/src/include/duckdb/catalog/catalog_entry/table_catalog_entry.hpp +0 -2
  39. package/src/duckdb/src/include/duckdb/catalog/catalog_entry/type_catalog_entry.hpp +1 -4
  40. package/src/duckdb/src/include/duckdb/catalog/catalog_set.hpp +0 -6
  41. package/src/duckdb/src/include/duckdb/common/extra_type_info.hpp +2 -6
  42. package/src/duckdb/src/include/duckdb/common/field_writer.hpp +0 -4
  43. package/src/duckdb/src/include/duckdb/common/file_system.hpp +8 -8
  44. package/src/duckdb/src/include/duckdb/common/filename_pattern.hpp +1 -1
  45. package/src/duckdb/src/include/duckdb/common/multi_file_reader_options.hpp +1 -1
  46. package/src/duckdb/src/include/duckdb/common/opener_file_system.hpp +4 -0
  47. package/src/duckdb/src/include/duckdb/common/serializer/buffered_file_reader.hpp +0 -4
  48. package/src/duckdb/src/include/duckdb/common/serializer.hpp +0 -7
  49. package/src/duckdb/src/include/duckdb/common/types/timestamp.hpp +14 -10
  50. package/src/duckdb/src/include/duckdb/common/types.hpp +5 -10
  51. package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp +2 -0
  52. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp +0 -4
  53. package/src/duckdb/src/include/duckdb/execution/physical_plan_generator.hpp +3 -2
  54. package/src/duckdb/src/include/duckdb/function/scalar_function.hpp +0 -3
  55. package/src/duckdb/src/include/duckdb/main/attached_database.hpp +1 -1
  56. package/src/duckdb/src/include/duckdb/optimizer/deliminator.hpp +2 -2
  57. package/src/duckdb/src/include/duckdb/planner/logical_tokens.hpp +0 -2
  58. package/src/duckdb/src/include/duckdb/planner/operator/list.hpp +1 -2
  59. package/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp +2 -4
  60. package/src/duckdb/src/include/duckdb/planner/operator/logical_create_index.hpp +10 -7
  61. package/src/duckdb/src/include/duckdb/planner/operator/logical_extension_operator.hpp +5 -0
  62. package/src/duckdb/src/include/duckdb/planner/operator_extension.hpp +1 -0
  63. package/src/duckdb/src/include/duckdb/storage/meta_block_reader.hpp +0 -3
  64. package/src/duckdb/src/include/duckdb.h +15 -0
  65. package/src/duckdb/src/main/attached_database.cpp +2 -2
  66. package/src/duckdb/src/main/capi/duckdb-c.cpp +16 -0
  67. package/src/duckdb/src/main/database.cpp +1 -1
  68. package/src/duckdb/src/main/db_instance_cache.cpp +14 -6
  69. package/src/duckdb/src/main/extension/extension_install.cpp +1 -1
  70. package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp +1 -2
  71. package/src/duckdb/src/optimizer/compressed_materialization.cpp +0 -1
  72. package/src/duckdb/src/optimizer/deliminator.cpp +7 -7
  73. package/src/duckdb/src/optimizer/unnest_rewriter.cpp +3 -5
  74. package/src/duckdb/src/parser/transform/helpers/transform_typename.cpp +16 -1
  75. package/src/duckdb/src/parser/transform/statement/transform_create_type.cpp +1 -1
  76. package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp +3 -3
  77. package/src/duckdb/src/planner/binder/statement/bind_create.cpp +8 -23
  78. package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +0 -24
  79. package/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp +3 -5
  80. package/src/duckdb/src/planner/logical_operator.cpp +1 -6
  81. package/src/duckdb/src/planner/logical_operator_visitor.cpp +3 -6
  82. package/src/duckdb/src/planner/operator/logical_comparison_join.cpp +2 -14
  83. package/src/duckdb/src/planner/operator/logical_create_index.cpp +21 -12
  84. package/src/duckdb/src/planner/operator/logical_extension_operator.cpp +24 -0
  85. package/src/duckdb/src/planner/planner.cpp +0 -1
  86. package/src/duckdb/src/storage/checkpoint_manager.cpp +1 -5
  87. package/src/duckdb/src/storage/meta_block_reader.cpp +0 -9
  88. package/src/duckdb/src/storage/serialization/serialize_create_info.cpp +2 -0
  89. package/src/duckdb/src/storage/serialization/serialize_logical_operator.cpp +27 -0
  90. package/src/duckdb/src/storage/storage_info.cpp +1 -1
  91. package/src/duckdb/src/storage/table/list_column_data.cpp +6 -3
  92. package/src/duckdb/src/storage/wal_replay.cpp +1 -3
  93. package/src/duckdb/ub_src_planner_operator.cpp +0 -4
  94. package/src/statement.cpp +0 -2
  95. package/test/columns.test.ts +1 -2
  96. package/src/duckdb/src/include/duckdb/planner/operator/logical_asof_join.hpp +0 -27
  97. package/src/duckdb/src/include/duckdb/planner/operator/logical_delim_join.hpp +0 -32
  98. package/src/duckdb/src/planner/operator/logical_asof_join.cpp +0 -14
  99. package/src/duckdb/src/planner/operator/logical_delim_join.cpp +0 -25
@@ -108,6 +108,10 @@ void VirtualFileSystem::RemoveFile(const string &filename) {
108
108
  FindFileSystem(filename).RemoveFile(filename);
109
109
  }
110
110
 
111
+ string VirtualFileSystem::PathSeparator(const string &path) {
112
+ return FindFileSystem(path).PathSeparator(path);
113
+ }
114
+
111
115
  vector<string> VirtualFileSystem::Glob(const string &path, FileOpener *opener) {
112
116
  return FindFileSystem(path).Glob(path, opener);
113
117
  }
@@ -39,7 +39,7 @@ static unique_ptr<FunctionData> UnionTagBind(ClientContext &context, ScalarFunct
39
39
  FlatVector::GetData<string_t>(varchar_vector)[i] =
40
40
  str.IsInlined() ? str : StringVector::AddString(varchar_vector, str);
41
41
  }
42
- auto enum_type = LogicalType::ENUM("", varchar_vector, member_count);
42
+ auto enum_type = LogicalType::ENUM(varchar_vector, member_count);
43
43
  bound_function.return_type = enum_type;
44
44
 
45
45
  return nullptr;
@@ -3,7 +3,6 @@
3
3
  #include "duckdb/planner/operator/logical_comparison_join.hpp"
4
4
  #include "duckdb/planner/operator/logical_any_join.hpp"
5
5
  #include "duckdb/planner/operator/logical_create_index.hpp"
6
- #include "duckdb/planner/operator/logical_delim_join.hpp"
7
6
  #include "duckdb/planner/operator/logical_insert.hpp"
8
7
 
9
8
  #include "duckdb/planner/expression/bound_columnref_expression.hpp"
@@ -29,12 +28,9 @@ void ColumnBindingResolver::VisitOperator(LogicalOperator &op) {
29
28
  for (auto &cond : comp_join.conditions) {
30
29
  VisitExpression(&cond.left);
31
30
  }
32
- if (op.type == LogicalOperatorType::LOGICAL_DELIM_JOIN) {
33
- // visit the duplicate eliminated columns on the LHS, if any
34
- auto &delim_join = op.Cast<LogicalDelimJoin>();
35
- for (auto &expr : delim_join.duplicate_eliminated_columns) {
36
- VisitExpression(&expr);
37
- }
31
+ // visit the duplicate eliminated columns on the LHS, if any
32
+ for (auto &expr : comp_join.duplicate_eliminated_columns) {
33
+ VisitExpression(&expr);
38
34
  }
39
35
  // then get the bindings of the RHS and resolve the RHS expressions
40
36
  VisitOperator(*comp_join.children[1]);
@@ -108,7 +108,7 @@ SinkCombineResultType PhysicalCopyToFile::Combine(ExecutionContext &context, Ope
108
108
  auto partition_key_map = l.part_buffer->GetReverseMap();
109
109
 
110
110
  string trimmed_path = file_path;
111
- StringUtil::RTrim(trimmed_path, fs.PathSeparator());
111
+ StringUtil::RTrim(trimmed_path, fs.PathSeparator(trimmed_path));
112
112
 
113
113
  for (idx_t i = 0; i < partitions.size(); i++) {
114
114
  string hive_path =
@@ -9,16 +9,6 @@
9
9
 
10
10
  namespace duckdb {
11
11
 
12
- PhysicalTableScan::PhysicalTableScan(vector<LogicalType> types, TableFunction function_p,
13
- unique_ptr<FunctionData> bind_data_p, vector<column_t> column_ids_p,
14
- vector<string> names_p, unique_ptr<TableFilterSet> table_filters_p,
15
- idx_t estimated_cardinality)
16
- : PhysicalOperator(PhysicalOperatorType::TABLE_SCAN, std::move(types), estimated_cardinality),
17
- function(std::move(function_p)), bind_data(std::move(bind_data_p)), column_ids(std::move(column_ids_p)),
18
- names(std::move(names_p)), table_filters(std::move(table_filters_p)) {
19
- extra_info.file_filters = "";
20
- }
21
-
22
12
  PhysicalTableScan::PhysicalTableScan(vector<LogicalType> types, TableFunction function_p,
23
13
  unique_ptr<FunctionData> bind_data_p, vector<LogicalType> returned_types_p,
24
14
  vector<column_t> column_ids_p, vector<idx_t> projection_ids_p,
@@ -66,7 +66,8 @@ SourceResultType PhysicalAttach::GetData(ExecutionContext &context, DataChunk &c
66
66
  const auto &path = info->path;
67
67
 
68
68
  if (name.empty()) {
69
- name = AttachedDatabase::ExtractDatabaseName(path);
69
+ auto &fs = FileSystem::GetFileSystem(context.client);
70
+ name = AttachedDatabase::ExtractDatabaseName(path, fs);
70
71
  }
71
72
  auto &db_manager = DatabaseManager::Get(context.client);
72
73
  auto existing_db = db_manager.GetDatabaseFromPath(context.client, path);
@@ -74,15 +74,11 @@ SourceResultType PhysicalCreateType::GetData(ExecutionContext &context, DataChun
74
74
  if (IsSink()) {
75
75
  D_ASSERT(info->type == LogicalType::INVALID);
76
76
  auto &g_sink_state = sink_state->Cast<CreateTypeGlobalState>();
77
- info->type = LogicalType::ENUM(info->name, g_sink_state.result, g_sink_state.size);
77
+ info->type = LogicalType::ENUM(g_sink_state.result, g_sink_state.size);
78
78
  }
79
79
 
80
80
  auto &catalog = Catalog::GetCatalog(context.client, info->catalog);
81
- auto catalog_entry = catalog.CreateType(context.client, *info);
82
- D_ASSERT(catalog_entry->type == CatalogType::TYPE_ENTRY);
83
- auto &catalog_type = catalog_entry->Cast<TypeCatalogEntry>();
84
- EnumType::SetCatalog(info->type, &catalog_type);
85
-
81
+ catalog.CreateType(context.client, *info);
86
82
  return SourceResultType::FINISHED;
87
83
  }
88
84
 
@@ -7,11 +7,10 @@
7
7
  #include "duckdb/planner/expression/bound_constant_expression.hpp"
8
8
  #include "duckdb/planner/expression/bound_reference_expression.hpp"
9
9
  #include "duckdb/planner/expression/bound_window_expression.hpp"
10
- #include "duckdb/planner/operator/logical_asof_join.hpp"
11
10
 
12
11
  namespace duckdb {
13
12
 
14
- unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalAsOfJoin &op) {
13
+ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::PlanAsOfJoin(LogicalComparisonJoin &op) {
15
14
  // now visit the children
16
15
  D_ASSERT(op.children.size() == 2);
17
16
  idx_t lhs_cardinality = op.children[0]->EstimateCardinality(context);
@@ -237,7 +237,7 @@ static void RewriteJoinCondition(Expression &expr, idx_t offset) {
237
237
  ExpressionIterator::EnumerateChildren(expr, [&](Expression &child) { RewriteJoinCondition(child, offset); });
238
238
  }
239
239
 
240
- unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalComparisonJoin &op) {
240
+ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::PlanComparisonJoin(LogicalComparisonJoin &op) {
241
241
  // now visit the children
242
242
  D_ASSERT(op.children.size() == 2);
243
243
  idx_t lhs_cardinality = op.children[0]->EstimateCardinality(context);
@@ -338,4 +338,17 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalComparison
338
338
  return plan;
339
339
  }
340
340
 
341
+ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalComparisonJoin &op) {
342
+ switch (op.type) {
343
+ case LogicalOperatorType::LOGICAL_ASOF_JOIN:
344
+ return PlanAsOfJoin(op);
345
+ case LogicalOperatorType::LOGICAL_COMPARISON_JOIN:
346
+ return PlanComparisonJoin(op);
347
+ case LogicalOperatorType::LOGICAL_DELIM_JOIN:
348
+ return PlanDelimJoin(op);
349
+ default:
350
+ throw InternalException("Unrecognized operator type for LogicalComparisonJoin");
351
+ }
352
+ }
353
+
341
354
  } // namespace duckdb
@@ -1,12 +1,11 @@
1
1
  #include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp"
2
2
  #include "duckdb/execution/operator/projection/physical_projection.hpp"
3
3
  #include "duckdb/execution/operator/filter/physical_filter.hpp"
4
- #include "duckdb/execution/operator/scan/physical_table_scan.hpp"
5
4
  #include "duckdb/execution/operator/schema/physical_create_index.hpp"
6
5
  #include "duckdb/execution/operator/order/physical_order.hpp"
7
6
  #include "duckdb/execution/physical_plan_generator.hpp"
8
- #include "duckdb/function/table/table_scan.hpp"
9
7
  #include "duckdb/planner/operator/logical_create_index.hpp"
8
+ #include "duckdb/planner/operator/logical_get.hpp"
10
9
  #include "duckdb/planner/expression/bound_operator_expression.hpp"
11
10
  #include "duckdb/planner/expression/bound_reference_expression.hpp"
12
11
  #include "duckdb/planner/table_filter.hpp"
@@ -14,11 +13,10 @@
14
13
  namespace duckdb {
15
14
 
16
15
  unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalCreateIndex &op) {
17
-
18
16
  // generate a physical plan for the parallel index creation which consists of the following operators
19
17
  // table scan - projection (for expression execution) - filter (NOT NULL) - order - create index
20
-
21
- D_ASSERT(op.children.empty());
18
+ D_ASSERT(op.children.size() == 1);
19
+ auto table_scan = CreatePlan(*op.children[0]);
22
20
 
23
21
  // validate that all expressions contain valid scalar functions
24
22
  // e.g. get_current_timestamp(), random(), and sequence values are not allowed as ART keys
@@ -32,19 +30,7 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalCreateInde
32
30
  }
33
31
 
34
32
  // table scan operator for index key columns and row IDs
35
-
36
- unique_ptr<TableFilterSet> table_filters;
37
- op.info->column_ids.emplace_back(COLUMN_IDENTIFIER_ROW_ID);
38
-
39
- auto &bind_data = op.bind_data->Cast<TableScanBindData>();
40
- bind_data.is_create_index = true;
41
-
42
- auto table_scan =
43
- make_uniq<PhysicalTableScan>(op.info->scan_types, op.function, std::move(op.bind_data), op.info->column_ids,
44
- op.info->names, std::move(table_filters), op.estimated_cardinality);
45
-
46
33
  dependencies.AddDependency(op.table);
47
- op.info->column_ids.pop_back();
48
34
 
49
35
  D_ASSERT(op.info->scan_types.size() - 1 <= op.info->names.size());
50
36
  D_ASSERT(op.info->scan_types.size() - 1 <= op.info->column_ids.size());
@@ -1,10 +1,7 @@
1
- #include "duckdb/execution/aggregate_hashtable.hpp"
2
1
  #include "duckdb/execution/operator/join/physical_delim_join.hpp"
3
2
  #include "duckdb/execution/operator/join/physical_hash_join.hpp"
4
3
  #include "duckdb/execution/operator/projection/physical_projection.hpp"
5
4
  #include "duckdb/execution/physical_plan_generator.hpp"
6
- #include "duckdb/planner/operator/logical_delim_join.hpp"
7
- #include "duckdb/planner/expression/bound_aggregate_expression.hpp"
8
5
  #include "duckdb/planner/expression/bound_reference_expression.hpp"
9
6
  #include "duckdb/execution/operator/aggregate/physical_hash_aggregate.hpp"
10
7
 
@@ -19,9 +16,9 @@ static void GatherDelimScans(const PhysicalOperator &op, vector<const_reference<
19
16
  }
20
17
  }
21
18
 
22
- unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalDelimJoin &op) {
19
+ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::PlanDelimJoin(LogicalComparisonJoin &op) {
23
20
  // first create the underlying join
24
- auto plan = CreatePlan(op.Cast<LogicalComparisonJoin>());
21
+ auto plan = PlanComparisonJoin(op);
25
22
  // this should create a join, not a cross product
26
23
  D_ASSERT(plan && plan->type != PhysicalOperatorType::CROSS_PRODUCT);
27
24
  // duplicate eliminated join
@@ -113,12 +113,8 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalOperator &
113
113
  case LogicalOperatorType::LOGICAL_ANY_JOIN:
114
114
  plan = CreatePlan(op.Cast<LogicalAnyJoin>());
115
115
  break;
116
- case LogicalOperatorType::LOGICAL_DELIM_JOIN:
117
- plan = CreatePlan(op.Cast<LogicalDelimJoin>());
118
- break;
119
116
  case LogicalOperatorType::LOGICAL_ASOF_JOIN:
120
- plan = CreatePlan(op.Cast<LogicalAsOfJoin>());
121
- break;
117
+ case LogicalOperatorType::LOGICAL_DELIM_JOIN:
122
118
  case LogicalOperatorType::LOGICAL_COMPARISON_JOIN:
123
119
  plan = CreatePlan(op.Cast<LogicalComparisonJoin>());
124
120
  break;
@@ -36,7 +36,6 @@ bool StringEnumCastLoop(const string_t *source_data, ValidityMask &source_mask,
36
36
  template <class T>
37
37
  bool StringEnumCast(Vector &source, Vector &result, idx_t count, CastParameters &parameters) {
38
38
  D_ASSERT(source.GetType().id() == LogicalTypeId::VARCHAR);
39
- auto enum_name = EnumType::GetTypeName(result.GetType());
40
39
  switch (source.GetVectorType()) {
41
40
  case VectorType::CONSTANT_VECTOR: {
42
41
  result.SetVectorType(VectorType::CONSTANT_VECTOR);
@@ -27,9 +27,10 @@ ScalarFunction::ScalarFunction(vector<LogicalType> arguments, LogicalType return
27
27
  }
28
28
 
29
29
  bool ScalarFunction::operator==(const ScalarFunction &rhs) const {
30
- return CompareScalarFunctionT(rhs.function) && bind == rhs.bind && dependency == rhs.dependency &&
31
- statistics == rhs.statistics;
30
+ return name == rhs.name && arguments == rhs.arguments && return_type == rhs.return_type && varargs == rhs.varargs &&
31
+ bind == rhs.bind && dependency == rhs.dependency && statistics == rhs.statistics;
32
32
  }
33
+
33
34
  bool ScalarFunction::operator!=(const ScalarFunction &rhs) const {
34
35
  return !(*this == rhs);
35
36
  }
@@ -57,23 +58,6 @@ bool ScalarFunction::Equal(const ScalarFunction &rhs) const {
57
58
  return true; // they are equal
58
59
  }
59
60
 
60
- bool ScalarFunction::CompareScalarFunctionT(const scalar_function_t &other) const {
61
- typedef void(scalar_function_ptr_t)(DataChunk &, ExpressionState &, Vector &);
62
-
63
- auto func_ptr = (scalar_function_ptr_t **)function.template target<scalar_function_ptr_t *>(); // NOLINT
64
- auto other_ptr = (scalar_function_ptr_t **)other.template target<scalar_function_ptr_t *>(); // NOLINT
65
-
66
- // Case the functions were created from lambdas the target will return a nullptr
67
- if (!func_ptr && !other_ptr) {
68
- return true;
69
- }
70
- if (func_ptr == nullptr || other_ptr == nullptr) {
71
- // scalar_function_t (std::functions) from lambdas cannot be compared
72
- return false;
73
- }
74
- return CastPointerToValue(*func_ptr) == CastPointerToValue(*other_ptr);
75
- }
76
-
77
61
  void ScalarFunction::NopFunction(DataChunk &input, ExpressionState &state, Vector &result) {
78
62
  D_ASSERT(input.ColumnCount() >= 1);
79
63
  result.Reference(input.data[0]);
@@ -123,7 +123,7 @@ LogicalType ArrowTableFunction::GetArrowLogicalType(
123
123
  child_list_t<LogicalType> child_types;
124
124
  for (idx_t type_idx = 0; type_idx < (idx_t)schema.n_children; type_idx++) {
125
125
  auto child_type = GetArrowLogicalType(*schema.children[type_idx], arrow_convert_data, col_idx);
126
- child_types.push_back({schema.children[type_idx]->name, child_type});
126
+ child_types.emplace_back(schema.children[type_idx]->name, child_type);
127
127
  }
128
128
  return LogicalType::STRUCT(child_types);
129
129
 
@@ -93,7 +93,7 @@ static void SetValidityMask(Vector &vector, ArrowArray &array, ArrowScanLocalSta
93
93
  static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowScanLocalState &scan_state, idx_t size,
94
94
  std::unordered_map<idx_t, unique_ptr<ArrowConvertData>> &arrow_convert_data,
95
95
  idx_t col_idx, ArrowConvertDataIndices &arrow_convert_idx, int64_t nested_offset = -1,
96
- ValidityMask *parent_mask = nullptr);
96
+ ValidityMask *parent_mask = nullptr, uint64_t parent_offset = 0);
97
97
 
98
98
  static void ArrowToDuckDBList(Vector &vector, ArrowArray &array, ArrowScanLocalState &scan_state, idx_t size,
99
99
  std::unordered_map<idx_t, unique_ptr<ArrowConvertData>> &arrow_convert_data,
@@ -265,12 +265,13 @@ static void SetVectorString(Vector &vector, idx_t size, char *cdata, T *offsets)
265
265
  }
266
266
  }
267
267
 
268
- static void DirectConversion(Vector &vector, ArrowArray &array, ArrowScanLocalState &scan_state,
269
- int64_t nested_offset) {
268
+ static void DirectConversion(Vector &vector, ArrowArray &array, ArrowScanLocalState &scan_state, int64_t nested_offset,
269
+ uint64_t parent_offset) {
270
270
  auto internal_type = GetTypeIdSize(vector.GetType().InternalType());
271
- auto data_ptr = ArrowBufferData<data_t>(array, 1) + internal_type * (scan_state.chunk_offset + array.offset);
271
+ auto data_ptr =
272
+ ArrowBufferData<data_t>(array, 1) + internal_type * (scan_state.chunk_offset + array.offset + parent_offset);
272
273
  if (nested_offset != -1) {
273
- data_ptr = ArrowBufferData<data_t>(array, 1) + internal_type * (array.offset + nested_offset);
274
+ data_ptr = ArrowBufferData<data_t>(array, 1) + internal_type * (array.offset + nested_offset + parent_offset);
274
275
  }
275
276
  FlatVector::SetData(vector, data_ptr);
276
277
  }
@@ -359,7 +360,7 @@ static void IntervalConversionMonthDayNanos(Vector &vector, ArrowArray &array, A
359
360
  static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowScanLocalState &scan_state, idx_t size,
360
361
  std::unordered_map<idx_t, unique_ptr<ArrowConvertData>> &arrow_convert_data,
361
362
  idx_t col_idx, ArrowConvertDataIndices &arrow_convert_idx, int64_t nested_offset,
362
- ValidityMask *parent_mask) {
363
+ ValidityMask *parent_mask, uint64_t parent_offset) {
363
364
  switch (vector.GetType().id()) {
364
365
  case LogicalTypeId::SQLNULL:
365
366
  vector.Reference(Value());
@@ -407,7 +408,7 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowScanLoca
407
408
  case LogicalTypeId::TIMESTAMP_SEC:
408
409
  case LogicalTypeId::TIMESTAMP_MS:
409
410
  case LogicalTypeId::TIMESTAMP_NS: {
410
- DirectConversion(vector, array, scan_state, nested_offset);
411
+ DirectConversion(vector, array, scan_state, nested_offset, parent_offset);
411
412
  break;
412
413
  }
413
414
  case LogicalTypeId::VARCHAR: {
@@ -432,7 +433,7 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowScanLoca
432
433
  auto precision = arrow_convert_data[col_idx]->date_time_precision[arrow_convert_idx.datetime_precision_index++];
433
434
  switch (precision) {
434
435
  case ArrowDateTimeType::DAYS: {
435
- DirectConversion(vector, array, scan_state, nested_offset);
436
+ DirectConversion(vector, array, scan_state, nested_offset, parent_offset);
436
437
  break;
437
438
  }
438
439
  case ArrowDateTimeType::MILLISECONDS: {
@@ -495,7 +496,7 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowScanLoca
495
496
  break;
496
497
  }
497
498
  case ArrowDateTimeType::MICROSECONDS: {
498
- DirectConversion(vector, array, scan_state, nested_offset);
499
+ DirectConversion(vector, array, scan_state, nested_offset, parent_offset);
499
500
  break;
500
501
  }
501
502
  case ArrowDateTimeType::NANOSECONDS: {
@@ -640,7 +641,8 @@ static void ColumnArrowToDuckDB(Vector &vector, ArrowArray &array, ArrowScanLoca
640
641
  }
641
642
  }
642
643
  ColumnArrowToDuckDB(*child_entries[type_idx], *array.children[type_idx], scan_state, size,
643
- arrow_convert_data, col_idx, arrow_convert_idx, nested_offset, &struct_validity_mask);
644
+ arrow_convert_data, col_idx, arrow_convert_idx, nested_offset, &struct_validity_mask,
645
+ array.offset);
644
646
  }
645
647
  break;
646
648
  }
@@ -1297,7 +1297,8 @@ unique_ptr<TableRef> ReadCSVReplacement(ClientContext &context, const string &ta
1297
1297
  table_function->function = make_uniq<FunctionExpression>("read_csv_auto", std::move(children));
1298
1298
 
1299
1299
  if (!FileSystem::HasGlob(table_name)) {
1300
- table_function->alias = FileSystem::ExtractBaseName(table_name);
1300
+ auto &fs = FileSystem::GetFileSystem(context);
1301
+ table_function->alias = fs.ExtractBaseName(table_name);
1301
1302
  }
1302
1303
 
1303
1304
  return std::move(table_function);
@@ -70,14 +70,14 @@ vector<TestType> TestAllTypesFun::GetTestTypes(bool use_large_enum) {
70
70
  auto small_enum_ptr = FlatVector::GetData<string_t>(small_enum);
71
71
  small_enum_ptr[0] = StringVector::AddStringOrBlob(small_enum, "DUCK_DUCK_ENUM");
72
72
  small_enum_ptr[1] = StringVector::AddStringOrBlob(small_enum, "GOOSE");
73
- result.emplace_back(LogicalType::ENUM("small_enum", small_enum, 2), "small_enum");
73
+ result.emplace_back(LogicalType::ENUM(small_enum, 2), "small_enum");
74
74
 
75
75
  Vector medium_enum(LogicalType::VARCHAR, 300);
76
76
  auto medium_enum_ptr = FlatVector::GetData<string_t>(medium_enum);
77
77
  for (idx_t i = 0; i < 300; i++) {
78
78
  medium_enum_ptr[i] = StringVector::AddStringOrBlob(medium_enum, string("enum_") + to_string(i));
79
79
  }
80
- result.emplace_back(LogicalType::ENUM("medium_enum", medium_enum, 300), "medium_enum");
80
+ result.emplace_back(LogicalType::ENUM(medium_enum, 300), "medium_enum");
81
81
 
82
82
  if (use_large_enum) {
83
83
  // this is a big one... not sure if we should push this one here, but it's required for completeness
@@ -86,13 +86,13 @@ vector<TestType> TestAllTypesFun::GetTestTypes(bool use_large_enum) {
86
86
  for (idx_t i = 0; i < 70000; i++) {
87
87
  large_enum_ptr[i] = StringVector::AddStringOrBlob(large_enum, string("enum_") + to_string(i));
88
88
  }
89
- result.emplace_back(LogicalType::ENUM("large_enum", large_enum, 70000), "large_enum");
89
+ result.emplace_back(LogicalType::ENUM(large_enum, 70000), "large_enum");
90
90
  } else {
91
91
  Vector large_enum(LogicalType::VARCHAR, 2);
92
92
  auto large_enum_ptr = FlatVector::GetData<string_t>(large_enum);
93
93
  large_enum_ptr[0] = StringVector::AddStringOrBlob(large_enum, string("enum_") + to_string(0));
94
94
  large_enum_ptr[1] = StringVector::AddStringOrBlob(large_enum, string("enum_") + to_string(69999));
95
- result.emplace_back(LogicalType::ENUM("large_enum", large_enum, 2), "large_enum");
95
+ result.emplace_back(LogicalType::ENUM(large_enum, 2), "large_enum");
96
96
  }
97
97
 
98
98
  // arrays
@@ -1,8 +1,8 @@
1
1
  #ifndef DUCKDB_VERSION
2
- #define DUCKDB_VERSION "0.8.2-dev2133"
2
+ #define DUCKDB_VERSION "0.8.2-dev2278"
3
3
  #endif
4
4
  #ifndef DUCKDB_SOURCE_ID
5
- #define DUCKDB_SOURCE_ID "726b6d1566"
5
+ #define DUCKDB_SOURCE_ID "a868375e85"
6
6
  #endif
7
7
  #include "duckdb/function/table/system_functions.hpp"
8
8
  #include "duckdb/main/database.hpp"
@@ -71,8 +71,6 @@ public:
71
71
  DUCKDB_API vector<LogicalType> GetTypes();
72
72
  //! Returns a list of the columns of the table
73
73
  DUCKDB_API const ColumnList &GetColumns() const;
74
- //! Returns a mutable list of the columns of the table
75
- DUCKDB_API ColumnList &GetColumnsMutable();
76
74
  //! Returns the underlying storage of the table
77
75
  virtual DataTable &GetStorage();
78
76
  //! Returns a list of the bound constraints of the table
@@ -29,10 +29,7 @@ public:
29
29
  LogicalType user_type;
30
30
 
31
31
  public:
32
- //! Serialize the meta information of the TypeCatalogEntry a serializer
33
- virtual void Serialize(Serializer &serializer) const;
34
- //! Deserializes to a TypeCatalogEntry
35
- static unique_ptr<CreateTypeInfo> Deserialize(Deserializer &source);
32
+ unique_ptr<CreateInfo> GetInfo() const override;
36
33
 
37
34
  string ToSQL() const override;
38
35
  };
@@ -125,12 +125,6 @@ public:
125
125
  void Verify(Catalog &catalog);
126
126
 
127
127
  private:
128
- //! Adjusts table dependencies on the event of an UNDO
129
- void AdjustTableDependencies(CatalogEntry &entry);
130
- //! Adjust one dependency
131
- void AdjustDependency(CatalogEntry &entry, TableCatalogEntry &table, ColumnDefinition &column, bool remove);
132
- //! Adjust User dependency
133
- void AdjustUserDependency(CatalogEntry &entry, ColumnDefinition &column, bool remove);
134
128
  //! Given a root entry, gets the entry valid for this transaction
135
129
  CatalogEntry &GetEntryForTransaction(CatalogTransaction transaction, CatalogEntry &current);
136
130
  CatalogEntry &GetCommittedEntry(CatalogEntry &current);
@@ -35,7 +35,6 @@ struct ExtraTypeInfo {
35
35
 
36
36
  ExtraTypeInfoType type;
37
37
  string alias;
38
- optional_ptr<TypeCatalogEntry> catalog_entry;
39
38
 
40
39
  public:
41
40
  bool Equals(ExtraTypeInfo *other_p) const;
@@ -184,19 +183,17 @@ private:
184
183
  enum EnumDictType : uint8_t { INVALID = 0, VECTOR_DICT = 1 };
185
184
 
186
185
  struct EnumTypeInfo : public ExtraTypeInfo {
187
- explicit EnumTypeInfo(string enum_name_p, Vector &values_insert_order_p, idx_t dict_size_p);
186
+ explicit EnumTypeInfo(Vector &values_insert_order_p, idx_t dict_size_p);
188
187
  EnumTypeInfo(const EnumTypeInfo &) = delete;
189
188
  EnumTypeInfo &operator=(const EnumTypeInfo &) = delete;
190
189
 
191
190
  public:
192
191
  const EnumDictType &GetEnumDictType() const;
193
- const string &GetEnumName() const;
194
- const string GetSchemaName() const;
195
192
  const Vector &GetValuesInsertOrder() const;
196
193
  const idx_t &GetDictSize() const;
197
194
  static PhysicalType DictType(idx_t size);
198
195
 
199
- static LogicalType CreateType(const string &enum_name, Vector &ordered_data, idx_t size);
196
+ static LogicalType CreateType(Vector &ordered_data, idx_t size);
200
197
 
201
198
  void Serialize(FieldWriter &writer) const override;
202
199
  static shared_ptr<ExtraTypeInfo> Deserialize(FieldReader &reader);
@@ -212,7 +209,6 @@ protected:
212
209
 
213
210
  private:
214
211
  EnumDictType dict_type;
215
- string enum_name;
216
212
  idx_t dict_size;
217
213
  };
218
214
 
@@ -167,10 +167,6 @@ public:
167
167
  return root.GetContext();
168
168
  }
169
169
 
170
- optional_ptr<Catalog> GetCatalog() override {
171
- return root.GetCatalog();
172
- }
173
-
174
170
  private:
175
171
  Deserializer &root;
176
172
  idx_t remaining_data;
@@ -185,21 +185,21 @@ public:
185
185
  DUCKDB_API virtual string ExpandPath(const string &path);
186
186
  //! Returns the system-available memory in bytes. Returns DConstants::INVALID_INDEX if the system function fails.
187
187
  DUCKDB_API static idx_t GetAvailableMemory();
188
- //! Path separator for the current file system
189
- DUCKDB_API static string PathSeparator();
188
+ //! Path separator for path
189
+ DUCKDB_API virtual string PathSeparator(const string &path);
190
190
  //! Checks if path is starts with separator (i.e., '/' on UNIX '\\' on Windows)
191
- DUCKDB_API static bool IsPathAbsolute(const string &path);
191
+ DUCKDB_API bool IsPathAbsolute(const string &path);
192
192
  //! Normalize an absolute path - the goal of normalizing is converting "\test.db" and "C:/test.db" into "C:\test.db"
193
193
  //! so that the database system cache can correctly
194
- DUCKDB_API static string NormalizeAbsolutePath(const string &path);
194
+ DUCKDB_API string NormalizeAbsolutePath(const string &path);
195
195
  //! Join two paths together
196
- DUCKDB_API static string JoinPath(const string &a, const string &path);
196
+ DUCKDB_API string JoinPath(const string &a, const string &path);
197
197
  //! Convert separators in a path to the local separators (e.g. convert "/" into \\ on windows)
198
- DUCKDB_API static string ConvertSeparators(const string &path);
198
+ DUCKDB_API string ConvertSeparators(const string &path);
199
199
  //! Extract the base name of a file (e.g. if the input is lib/example.dll the base name is 'example')
200
- DUCKDB_API static string ExtractBaseName(const string &path);
200
+ DUCKDB_API string ExtractBaseName(const string &path);
201
201
  //! Extract the name of a file (e.g if the input is lib/example.dll the name is 'example.dll')
202
- DUCKDB_API static string ExtractName(const string &path);
202
+ DUCKDB_API string ExtractName(const string &path);
203
203
 
204
204
  //! Returns the value of an environment variable - or the empty string if it is not set
205
205
  DUCKDB_API static string GetEnvVariable(const string &name);
@@ -23,7 +23,7 @@ public:
23
23
 
24
24
  public:
25
25
  void SetFilenamePattern(const string &pattern);
26
- string CreateFilename(const FileSystem &fs, const string &path, const string &extension, idx_t offset) const;
26
+ string CreateFilename(FileSystem &fs, const string &path, const string &extension, idx_t offset) const;
27
27
 
28
28
  private:
29
29
  string _base;
@@ -32,7 +32,7 @@ struct MultiFileReaderOptions {
32
32
  DUCKDB_API static MultiFileReaderOptions FormatDeserialize(FormatDeserializer &source);
33
33
  DUCKDB_API void AddBatchInfo(BindInfo &bind_info) const;
34
34
  DUCKDB_API void AutoDetectHivePartitioning(const vector<string> &files, ClientContext &context);
35
- DUCKDB_API static bool AutoDetectHivePartitioningInternal(const vector<string> &files);
35
+ DUCKDB_API static bool AutoDetectHivePartitioningInternal(const vector<string> &files, ClientContext &context);
36
36
  DUCKDB_API void AutoDetectHiveTypesInternal(const string &file, ClientContext &context);
37
37
  DUCKDB_API void VerifyHiveTypesArePartitions(const std::map<string, string> &partitions) const;
38
38
  DUCKDB_API LogicalType GetHiveLogicalType(const string &hive_partition_column) const;
@@ -103,6 +103,10 @@ public:
103
103
  GetFileSystem().RemoveFile(filename);
104
104
  }
105
105
 
106
+ string PathSeparator(const string &path) override {
107
+ return GetFileSystem().PathSeparator(path);
108
+ }
109
+
106
110
  vector<string> Glob(const string &path, FileOpener *opener = nullptr) override {
107
111
  if (opener) {
108
112
  throw InternalException("OpenerFileSystem cannot take an opener - the opener is pushed automatically");
@@ -37,14 +37,10 @@ public:
37
37
 
38
38
  ClientContext &GetContext() override;
39
39
 
40
- optional_ptr<Catalog> GetCatalog() override;
41
- void SetCatalog(Catalog &catalog);
42
-
43
40
  private:
44
41
  idx_t file_size;
45
42
  idx_t total_read;
46
43
  optional_ptr<ClientContext> context;
47
- optional_ptr<Catalog> catalog;
48
44
  };
49
45
 
50
46
  } // namespace duckdb
@@ -22,8 +22,6 @@ private:
22
22
  uint64_t version = 0L;
23
23
 
24
24
  public:
25
- bool is_query_plan = false;
26
-
27
25
  virtual ~Serializer() {
28
26
  }
29
27
 
@@ -119,11 +117,6 @@ public:
119
117
  throw InternalException("This deserializer does not have a client-context");
120
118
  };
121
119
 
122
- //! Gets the catalog for the deserializer
123
- virtual optional_ptr<Catalog> GetCatalog() {
124
- return nullptr;
125
- };
126
-
127
120
  template <class T>
128
121
  T Read() {
129
122
  T value;