duckdb 1.4.0 → 1.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (144) hide show
  1. package/package.json +1 -1
  2. package/src/duckdb/extension/core_functions/scalar/generic/current_setting.cpp +1 -4
  3. package/src/duckdb/extension/icu/icu-strptime.cpp +2 -1
  4. package/src/duckdb/extension/json/include/json_common.hpp +2 -4
  5. package/src/duckdb/extension/json/json_functions.cpp +5 -1
  6. package/src/duckdb/extension/parquet/column_writer.cpp +31 -21
  7. package/src/duckdb/extension/parquet/geo_parquet.cpp +21 -6
  8. package/src/duckdb/extension/parquet/include/column_writer.hpp +2 -2
  9. package/src/duckdb/extension/parquet/include/geo_parquet.hpp +28 -1
  10. package/src/duckdb/extension/parquet/include/parquet_writer.hpp +7 -2
  11. package/src/duckdb/extension/parquet/include/reader/string_column_reader.hpp +13 -0
  12. package/src/duckdb/extension/parquet/include/writer/array_column_writer.hpp +4 -0
  13. package/src/duckdb/extension/parquet/parquet_extension.cpp +56 -1
  14. package/src/duckdb/extension/parquet/parquet_reader.cpp +4 -1
  15. package/src/duckdb/extension/parquet/parquet_statistics.cpp +5 -7
  16. package/src/duckdb/extension/parquet/parquet_writer.cpp +15 -8
  17. package/src/duckdb/extension/parquet/reader/string_column_reader.cpp +17 -4
  18. package/src/duckdb/extension/parquet/writer/array_column_writer.cpp +22 -28
  19. package/src/duckdb/extension/parquet/writer/primitive_column_writer.cpp +17 -5
  20. package/src/duckdb/extension/parquet/writer/struct_column_writer.cpp +3 -2
  21. package/src/duckdb/src/catalog/catalog_search_path.cpp +2 -2
  22. package/src/duckdb/src/catalog/catalog_set.cpp +1 -2
  23. package/src/duckdb/src/common/enum_util.cpp +20 -0
  24. package/src/duckdb/src/common/file_system.cpp +0 -30
  25. package/src/duckdb/src/common/sorting/sort.cpp +25 -6
  26. package/src/duckdb/src/common/sorting/sorted_run_merger.cpp +1 -0
  27. package/src/duckdb/src/common/string_util.cpp +24 -0
  28. package/src/duckdb/src/common/virtual_file_system.cpp +59 -10
  29. package/src/duckdb/src/execution/index/art/art_merger.cpp +0 -3
  30. package/src/duckdb/src/execution/index/art/prefix.cpp +4 -0
  31. package/src/duckdb/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +1 -1
  32. package/src/duckdb/src/execution/operator/helper/physical_reset.cpp +2 -2
  33. package/src/duckdb/src/execution/operator/schema/physical_attach.cpp +1 -1
  34. package/src/duckdb/src/execution/physical_plan/plan_asof_join.cpp +3 -3
  35. package/src/duckdb/src/function/table/system/duckdb_connection_count.cpp +45 -0
  36. package/src/duckdb/src/function/table/system/duckdb_settings.cpp +11 -1
  37. package/src/duckdb/src/function/table/system_functions.cpp +1 -0
  38. package/src/duckdb/src/function/table/version/pragma_version.cpp +3 -3
  39. package/src/duckdb/src/include/duckdb/common/enum_util.hpp +8 -0
  40. package/src/duckdb/src/include/duckdb/common/string_util.hpp +2 -0
  41. package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp +4 -1
  42. package/src/duckdb/src/include/duckdb/function/scalar/variant_functions.hpp +1 -1
  43. package/src/duckdb/src/include/duckdb/function/table/system_functions.hpp +4 -0
  44. package/src/duckdb/src/include/duckdb/logging/log_storage.hpp +6 -6
  45. package/src/duckdb/src/include/duckdb/logging/log_type.hpp +26 -3
  46. package/src/duckdb/src/include/duckdb/main/attached_database.hpp +4 -0
  47. package/src/duckdb/src/include/duckdb/main/client_context.hpp +2 -0
  48. package/src/duckdb/src/include/duckdb/main/connection.hpp +0 -1
  49. package/src/duckdb/src/include/duckdb/main/connection_manager.hpp +0 -1
  50. package/src/duckdb/src/include/duckdb/main/database_file_path_manager.hpp +12 -1
  51. package/src/duckdb/src/include/duckdb/main/database_manager.hpp +3 -0
  52. package/src/duckdb/src/include/duckdb/main/relation/create_table_relation.hpp +2 -0
  53. package/src/duckdb/src/include/duckdb/main/relation/create_view_relation.hpp +2 -0
  54. package/src/duckdb/src/include/duckdb/main/relation/delete_relation.hpp +2 -0
  55. package/src/duckdb/src/include/duckdb/main/relation/explain_relation.hpp +2 -0
  56. package/src/duckdb/src/include/duckdb/main/relation/insert_relation.hpp +2 -0
  57. package/src/duckdb/src/include/duckdb/main/relation/query_relation.hpp +1 -0
  58. package/src/duckdb/src/include/duckdb/main/relation/update_relation.hpp +2 -0
  59. package/src/duckdb/src/include/duckdb/main/relation/write_csv_relation.hpp +2 -0
  60. package/src/duckdb/src/include/duckdb/main/relation/write_parquet_relation.hpp +2 -0
  61. package/src/duckdb/src/include/duckdb/main/relation.hpp +2 -1
  62. package/src/duckdb/src/include/duckdb/main/secret/secret.hpp +3 -1
  63. package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp +3 -2
  64. package/src/duckdb/src/include/duckdb/planner/binder.hpp +62 -3
  65. package/src/duckdb/src/include/duckdb/planner/expression_binder/lateral_binder.hpp +2 -2
  66. package/src/duckdb/src/include/duckdb/planner/operator/logical_cte.hpp +1 -1
  67. package/src/duckdb/src/include/duckdb/planner/operator/logical_dependent_join.hpp +3 -3
  68. package/src/duckdb/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp +2 -2
  69. package/src/duckdb/src/include/duckdb/planner/subquery/has_correlated_expressions.hpp +2 -2
  70. package/src/duckdb/src/include/duckdb/planner/subquery/rewrite_cte_scan.hpp +2 -2
  71. package/src/duckdb/src/include/duckdb/planner/tableref/bound_joinref.hpp +1 -1
  72. package/src/duckdb/src/include/duckdb/storage/compression/alp/alp_analyze.hpp +6 -1
  73. package/src/duckdb/src/include/duckdb/storage/compression/alprd/alprd_analyze.hpp +5 -1
  74. package/src/duckdb/src/include/duckdb/storage/metadata/metadata_manager.hpp +9 -7
  75. package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp +2 -0
  76. package/src/duckdb/src/include/duckdb/storage/table/array_column_data.hpp +4 -4
  77. package/src/duckdb/src/include/duckdb/storage/table/column_data.hpp +6 -6
  78. package/src/duckdb/src/include/duckdb/storage/table/list_column_data.hpp +4 -4
  79. package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +4 -4
  80. package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +5 -3
  81. package/src/duckdb/src/include/duckdb/storage/table/row_id_column_data.hpp +4 -4
  82. package/src/duckdb/src/include/duckdb/storage/table/standard_column_data.hpp +4 -4
  83. package/src/duckdb/src/include/duckdb/storage/table/struct_column_data.hpp +4 -4
  84. package/src/duckdb/src/include/duckdb/storage/table/update_segment.hpp +2 -2
  85. package/src/duckdb/src/include/duckdb/transaction/duck_transaction.hpp +2 -1
  86. package/src/duckdb/src/include/duckdb/transaction/update_info.hpp +4 -1
  87. package/src/duckdb/src/include/duckdb/transaction/wal_write_state.hpp +1 -1
  88. package/src/duckdb/src/logging/log_types.cpp +63 -0
  89. package/src/duckdb/src/main/attached_database.cpp +16 -3
  90. package/src/duckdb/src/main/client_context.cpp +27 -19
  91. package/src/duckdb/src/main/connection.cpp +2 -5
  92. package/src/duckdb/src/main/database_file_path_manager.cpp +23 -6
  93. package/src/duckdb/src/main/database_manager.cpp +18 -3
  94. package/src/duckdb/src/main/http/http_util.cpp +3 -1
  95. package/src/duckdb/src/main/relation/create_table_relation.cpp +8 -0
  96. package/src/duckdb/src/main/relation/create_view_relation.cpp +8 -0
  97. package/src/duckdb/src/main/relation/delete_relation.cpp +8 -0
  98. package/src/duckdb/src/main/relation/explain_relation.cpp +8 -0
  99. package/src/duckdb/src/main/relation/insert_relation.cpp +8 -0
  100. package/src/duckdb/src/main/relation/query_relation.cpp +4 -0
  101. package/src/duckdb/src/main/relation/update_relation.cpp +8 -0
  102. package/src/duckdb/src/main/relation/write_csv_relation.cpp +8 -0
  103. package/src/duckdb/src/main/relation/write_parquet_relation.cpp +8 -0
  104. package/src/duckdb/src/main/relation.cpp +2 -2
  105. package/src/duckdb/src/optimizer/filter_combiner.cpp +7 -0
  106. package/src/duckdb/src/optimizer/filter_pushdown.cpp +9 -3
  107. package/src/duckdb/src/optimizer/pushdown/pushdown_get.cpp +4 -1
  108. package/src/duckdb/src/optimizer/rule/comparison_simplification.cpp +3 -7
  109. package/src/duckdb/src/parser/statement/relation_statement.cpp +1 -4
  110. package/src/duckdb/src/parser/transform/statement/transform_create_function.cpp +2 -0
  111. package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp +8 -6
  112. package/src/duckdb/src/planner/binder/statement/bind_create.cpp +1 -5
  113. package/src/duckdb/src/planner/binder/statement/bind_merge_into.cpp +10 -2
  114. package/src/duckdb/src/planner/binder/statement/bind_pragma.cpp +20 -3
  115. package/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp +8 -3
  116. package/src/duckdb/src/planner/binder/tableref/bind_table_function.cpp +9 -2
  117. package/src/duckdb/src/planner/binder.cpp +2 -2
  118. package/src/duckdb/src/planner/expression_binder/lateral_binder.cpp +9 -13
  119. package/src/duckdb/src/planner/expression_binder/table_function_binder.cpp +4 -0
  120. package/src/duckdb/src/planner/expression_binder.cpp +3 -1
  121. package/src/duckdb/src/planner/operator/logical_dependent_join.cpp +2 -2
  122. package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp +12 -14
  123. package/src/duckdb/src/planner/subquery/has_correlated_expressions.cpp +1 -1
  124. package/src/duckdb/src/planner/subquery/rewrite_cte_scan.cpp +2 -2
  125. package/src/duckdb/src/storage/compression/bitpacking.cpp +1 -2
  126. package/src/duckdb/src/storage/data_table.cpp +2 -2
  127. package/src/duckdb/src/storage/local_storage.cpp +1 -1
  128. package/src/duckdb/src/storage/metadata/metadata_manager.cpp +67 -25
  129. package/src/duckdb/src/storage/statistics/string_stats.cpp +8 -0
  130. package/src/duckdb/src/storage/table/array_column_data.cpp +6 -5
  131. package/src/duckdb/src/storage/table/column_data.cpp +23 -9
  132. package/src/duckdb/src/storage/table/column_data_checkpointer.cpp +15 -1
  133. package/src/duckdb/src/storage/table/list_column_data.cpp +5 -4
  134. package/src/duckdb/src/storage/table/row_group.cpp +8 -8
  135. package/src/duckdb/src/storage/table/row_group_collection.cpp +12 -8
  136. package/src/duckdb/src/storage/table/row_id_column_data.cpp +5 -4
  137. package/src/duckdb/src/storage/table/standard_column_data.cpp +9 -8
  138. package/src/duckdb/src/storage/table/struct_column_data.cpp +10 -9
  139. package/src/duckdb/src/storage/table/update_segment.cpp +12 -10
  140. package/src/duckdb/src/transaction/commit_state.cpp +18 -0
  141. package/src/duckdb/src/transaction/duck_transaction.cpp +2 -10
  142. package/src/duckdb/src/transaction/wal_write_state.cpp +5 -5
  143. package/src/duckdb/third_party/httplib/httplib.hpp +6 -1
  144. package/src/duckdb/ub_src_function_table_system.cpp +2 -0
@@ -9,6 +9,22 @@ void ArrayColumnWriter::Analyze(ColumnWriterState &state_p, ColumnWriterState *p
9
9
  child_writer->Analyze(*state.child_state, &state_p, array_child, array_size * count);
10
10
  }
11
11
 
12
+ void ArrayColumnWriter::WriteArrayState(ListColumnWriterState &state, idx_t array_size, uint16_t first_repeat_level,
13
+ idx_t define_value, const bool is_empty) {
14
+ state.definition_levels.push_back(define_value);
15
+ state.repetition_levels.push_back(first_repeat_level);
16
+ state.is_empty.push_back(is_empty);
17
+
18
+ if (is_empty) {
19
+ return;
20
+ }
21
+ for (idx_t k = 1; k < array_size; k++) {
22
+ state.repetition_levels.push_back(MaxRepeat() + 1);
23
+ state.definition_levels.push_back(define_value);
24
+ state.is_empty.push_back(false);
25
+ }
26
+ }
27
+
12
28
  void ArrayColumnWriter::Prepare(ColumnWriterState &state_p, ColumnWriterState *parent, Vector &vector, idx_t count,
13
29
  bool vector_can_span_multiple_pages) {
14
30
  auto &state = state_p.Cast<ListColumnWriterState>();
@@ -25,42 +41,20 @@ void ArrayColumnWriter::Prepare(ColumnWriterState &state_p, ColumnWriterState *p
25
41
  for (idx_t i = start; i < vcount; i++) {
26
42
  idx_t parent_index = state.parent_index + i;
27
43
  if (parent && !parent->is_empty.empty() && parent->is_empty[parent_index]) {
28
- state.definition_levels.push_back(parent->definition_levels[parent_index]);
29
- state.repetition_levels.push_back(parent->repetition_levels[parent_index]);
30
- state.is_empty.push_back(true);
44
+ WriteArrayState(state, array_size, parent->repetition_levels[parent_index],
45
+ parent->definition_levels[parent_index], true);
31
46
  continue;
32
47
  }
33
48
  auto first_repeat_level =
34
49
  parent && !parent->repetition_levels.empty() ? parent->repetition_levels[parent_index] : MaxRepeat();
35
50
  if (parent && parent->definition_levels[parent_index] != PARQUET_DEFINE_VALID) {
36
- state.definition_levels.push_back(parent->definition_levels[parent_index]);
37
- state.repetition_levels.push_back(first_repeat_level);
38
- state.is_empty.push_back(false);
39
- for (idx_t k = 1; k < array_size; k++) {
40
- state.repetition_levels.push_back(MaxRepeat() + 1);
41
- state.definition_levels.push_back(parent->definition_levels[parent_index]);
42
- state.is_empty.push_back(false);
43
- }
51
+ WriteArrayState(state, array_size, first_repeat_level, parent->definition_levels[parent_index]);
44
52
  } else if (validity.RowIsValid(vector_index)) {
45
53
  // push the repetition levels
46
- state.definition_levels.push_back(PARQUET_DEFINE_VALID);
47
- state.is_empty.push_back(false);
48
-
49
- state.repetition_levels.push_back(first_repeat_level);
50
- for (idx_t k = 1; k < array_size; k++) {
51
- state.repetition_levels.push_back(MaxRepeat() + 1);
52
- state.definition_levels.push_back(PARQUET_DEFINE_VALID);
53
- state.is_empty.push_back(false);
54
- }
54
+ WriteArrayState(state, array_size, first_repeat_level, PARQUET_DEFINE_VALID);
55
55
  } else {
56
- state.definition_levels.push_back(MaxDefine() - 1);
57
- state.repetition_levels.push_back(first_repeat_level);
58
- state.is_empty.push_back(false);
59
- for (idx_t k = 1; k < array_size; k++) {
60
- state.repetition_levels.push_back(MaxRepeat() + 1);
61
- state.definition_levels.push_back(MaxDefine() - 1);
62
- state.is_empty.push_back(false);
63
- }
56
+ //! Produce a null
57
+ WriteArrayState(state, array_size, first_repeat_level, MaxDefine() - 1);
64
58
  }
65
59
  vector_index++;
66
60
  }
@@ -304,12 +304,24 @@ void PrimitiveColumnWriter::SetParquetStatistics(PrimitiveColumnWriterState &sta
304
304
  }
305
305
 
306
306
  if (state.stats_state->HasGeoStats()) {
307
- column_chunk.meta_data.__isset.geospatial_statistics = true;
308
- state.stats_state->WriteGeoStats(column_chunk.meta_data.geospatial_statistics);
309
307
 
310
- // Add the geospatial statistics to the extra GeoParquet metadata
311
- writer.GetGeoParquetData().AddGeoParquetStats(column_schema.name, column_schema.type,
312
- *state.stats_state->GetGeoStats());
308
+ auto gpq_version = writer.GetGeoParquetVersion();
309
+
310
+ const auto has_real_stats = gpq_version == GeoParquetVersion::NONE || gpq_version == GeoParquetVersion::BOTH ||
311
+ gpq_version == GeoParquetVersion::V2;
312
+ const auto has_json_stats = gpq_version == GeoParquetVersion::V1 || gpq_version == GeoParquetVersion::BOTH ||
313
+ gpq_version == GeoParquetVersion::V2;
314
+
315
+ if (has_real_stats) {
316
+ // Write the parquet native geospatial statistics
317
+ column_chunk.meta_data.__isset.geospatial_statistics = true;
318
+ state.stats_state->WriteGeoStats(column_chunk.meta_data.geospatial_statistics);
319
+ }
320
+ if (has_json_stats) {
321
+ // Add the geospatial statistics to the extra GeoParquet metadata
322
+ writer.GetGeoParquetData().AddGeoParquetStats(column_schema.name, column_schema.type,
323
+ *state.stats_state->GetGeoStats());
324
+ }
313
325
  }
314
326
 
315
327
  for (const auto &write_info : state.write_info) {
@@ -62,8 +62,9 @@ void StructColumnWriter::Prepare(ColumnWriterState &state_p, ColumnWriterState *
62
62
  auto &validity = FlatVector::Validity(vector);
63
63
  if (parent) {
64
64
  // propagate empty entries from the parent
65
- while (state.is_empty.size() < parent->is_empty.size()) {
66
- state.is_empty.push_back(parent->is_empty[state.is_empty.size()]);
65
+ if (state.is_empty.size() < parent->is_empty.size()) {
66
+ state.is_empty.insert(state.is_empty.end(), parent->is_empty.begin() + state.is_empty.size(),
67
+ parent->is_empty.end());
67
68
  }
68
69
  }
69
70
  HandleRepeatLevels(state_p, parent, count, MaxRepeat());
@@ -24,8 +24,8 @@ string CatalogSearchEntry::ToString() const {
24
24
 
25
25
  string CatalogSearchEntry::WriteOptionallyQuoted(const string &input) {
26
26
  for (idx_t i = 0; i < input.size(); i++) {
27
- if (input[i] == '.' || input[i] == ',') {
28
- return "\"" + input + "\"";
27
+ if (input[i] == '.' || input[i] == ',' || input[i] == '"') {
28
+ return "\"" + StringUtil::Replace(input, "\"", "\"\"") + "\"";
29
29
  }
30
30
  }
31
31
  return input;
@@ -401,8 +401,6 @@ bool CatalogSet::DropEntryInternal(CatalogTransaction transaction, const string
401
401
  throw CatalogException("Cannot drop entry \"%s\" because it is an internal system entry", entry->name);
402
402
  }
403
403
 
404
- entry->OnDrop();
405
-
406
404
  // create a new tombstone entry and replace the currently stored one
407
405
  // set the timestamp to the timestamp of the current transaction
408
406
  // and point it at the tombstone node
@@ -454,6 +452,7 @@ void CatalogSet::VerifyExistenceOfDependency(transaction_t commit_id, CatalogEnt
454
452
  void CatalogSet::CommitDrop(transaction_t commit_id, transaction_t start_time, CatalogEntry &entry) {
455
453
  auto &duck_catalog = GetCatalog();
456
454
 
455
+ entry.OnDrop();
457
456
  // Make sure that we don't see any uncommitted changes
458
457
  auto transaction_id = MAX_TRANSACTION_ID;
459
458
  // This will allow us to see all committed changes made before this COMMIT happened
@@ -117,6 +117,7 @@
117
117
  #include "duckdb/function/table/arrow/enum/arrow_variable_size_type.hpp"
118
118
  #include "duckdb/function/table_function.hpp"
119
119
  #include "duckdb/function/window/window_merge_sort_tree.hpp"
120
+ #include "duckdb/logging/log_storage.hpp"
120
121
  #include "duckdb/logging/logging.hpp"
121
122
  #include "duckdb/main/appender.hpp"
122
123
  #include "duckdb/main/capi/capi_internal.hpp"
@@ -2439,6 +2440,25 @@ LogMode EnumUtil::FromString<LogMode>(const char *value) {
2439
2440
  return static_cast<LogMode>(StringUtil::StringToEnum(GetLogModeValues(), 3, "LogMode", value));
2440
2441
  }
2441
2442
 
2443
+ const StringUtil::EnumStringLiteral *GetLoggingTargetTableValues() {
2444
+ static constexpr StringUtil::EnumStringLiteral values[] {
2445
+ { static_cast<uint32_t>(LoggingTargetTable::ALL_LOGS), "ALL_LOGS" },
2446
+ { static_cast<uint32_t>(LoggingTargetTable::LOG_ENTRIES), "LOG_ENTRIES" },
2447
+ { static_cast<uint32_t>(LoggingTargetTable::LOG_CONTEXTS), "LOG_CONTEXTS" }
2448
+ };
2449
+ return values;
2450
+ }
2451
+
2452
+ template<>
2453
+ const char* EnumUtil::ToChars<LoggingTargetTable>(LoggingTargetTable value) {
2454
+ return StringUtil::EnumToString(GetLoggingTargetTableValues(), 3, "LoggingTargetTable", static_cast<uint32_t>(value));
2455
+ }
2456
+
2457
+ template<>
2458
+ LoggingTargetTable EnumUtil::FromString<LoggingTargetTable>(const char *value) {
2459
+ return static_cast<LoggingTargetTable>(StringUtil::StringToEnum(GetLoggingTargetTableValues(), 3, "LoggingTargetTable", value));
2460
+ }
2461
+
2442
2462
  const StringUtil::EnumStringLiteral *GetLogicalOperatorTypeValues() {
2443
2463
  static constexpr StringUtil::EnumStringLiteral values[] {
2444
2464
  { static_cast<uint32_t>(LogicalOperatorType::LOGICAL_INVALID), "LOGICAL_INVALID" },
@@ -628,39 +628,9 @@ bool FileSystem::CanHandleFile(const string &fpath) {
628
628
  throw NotImplementedException("%s: CanHandleFile is not implemented!", GetName());
629
629
  }
630
630
 
631
- static string LookupExtensionForPattern(const string &pattern) {
632
- for (const auto &entry : EXTENSION_FILE_PREFIXES) {
633
- if (StringUtil::StartsWith(pattern, entry.name)) {
634
- return entry.extension;
635
- }
636
- }
637
- return "";
638
- }
639
-
640
631
  vector<OpenFileInfo> FileSystem::GlobFiles(const string &pattern, ClientContext &context, const FileGlobInput &input) {
641
632
  auto result = Glob(pattern);
642
633
  if (result.empty()) {
643
- string required_extension = LookupExtensionForPattern(pattern);
644
- if (!required_extension.empty() && !context.db->ExtensionIsLoaded(required_extension)) {
645
- auto &dbconfig = DBConfig::GetConfig(context);
646
- if (!ExtensionHelper::CanAutoloadExtension(required_extension) ||
647
- !dbconfig.options.autoload_known_extensions) {
648
- auto error_message =
649
- "File " + pattern + " requires the extension " + required_extension + " to be loaded";
650
- error_message =
651
- ExtensionHelper::AddExtensionInstallHintToErrorMsg(context, error_message, required_extension);
652
- throw MissingExtensionException(error_message);
653
- }
654
- // an extension is required to read this file, but it is not loaded - try to load it
655
- ExtensionHelper::AutoLoadExtension(context, required_extension);
656
- // success! glob again
657
- // check the extension is loaded just in case to prevent an infinite loop here
658
- if (!context.db->ExtensionIsLoaded(required_extension)) {
659
- throw InternalException("Extension load \"%s\" did not throw but somehow the extension was not loaded",
660
- required_extension);
661
- }
662
- return GlobFiles(pattern, context, input);
663
- }
664
634
  if (input.behavior == FileGlobOptions::FALLBACK_GLOB && !HasGlob(pattern)) {
665
635
  // if we have no glob in the pattern and we have an extension, we try to glob
666
636
  if (!HasGlob(pattern)) {
@@ -378,6 +378,15 @@ public:
378
378
  return merger_global_state ? merger_global_state->MaxThreads() : 1;
379
379
  }
380
380
 
381
+ void Destroy() {
382
+ if (!merger_global_state) {
383
+ return;
384
+ }
385
+ auto guard = merger_global_state->Lock();
386
+ merger.sorted_runs.clear();
387
+ sink.temporary_memory_state.reset();
388
+ }
389
+
381
390
  public:
382
391
  //! The global sink state
383
392
  SortGlobalSinkState &sink;
@@ -477,16 +486,26 @@ SourceResultType Sort::MaterializeColumnData(ExecutionContext &context, Operator
477
486
  }
478
487
 
479
488
  // Merge into global output collection
480
- auto guard = gstate.Lock();
481
- if (!gstate.column_data) {
482
- gstate.column_data = std::move(local_column_data);
483
- } else {
484
- gstate.column_data->Merge(*local_column_data);
489
+ {
490
+ auto guard = gstate.Lock();
491
+ if (!gstate.column_data) {
492
+ gstate.column_data = std::move(local_column_data);
493
+ } else {
494
+ gstate.column_data->Merge(*local_column_data);
495
+ }
485
496
  }
486
497
 
498
+ // Destroy local state before returning
499
+ input.local_state.Cast<SortLocalSourceState>().merger_local_state.reset();
500
+
487
501
  // Return type indicates whether materialization is done
488
502
  const auto progress_data = GetProgress(context.client, input.global_state);
489
- return progress_data.done == progress_data.total ? SourceResultType::FINISHED : SourceResultType::HAVE_MORE_OUTPUT;
503
+ if (progress_data.done == progress_data.total) {
504
+ // Destroy global state before returning
505
+ gstate.Destroy();
506
+ return SourceResultType::FINISHED;
507
+ }
508
+ return SourceResultType::HAVE_MORE_OUTPUT;
490
509
  }
491
510
 
492
511
  unique_ptr<ColumnDataCollection> Sort::GetColumnData(OperatorSourceInput &input) const {
@@ -888,6 +888,7 @@ SortedRunMerger::SortedRunMerger(const Expression &decode_sort_key_p, shared_ptr
888
888
  unique_ptr<LocalSourceState> SortedRunMerger::GetLocalSourceState(ExecutionContext &,
889
889
  GlobalSourceState &gstate_p) const {
890
890
  auto &gstate = gstate_p.Cast<SortedRunMergerGlobalState>();
891
+ auto guard = gstate.Lock();
891
892
  return make_uniq<SortedRunMergerLocalState>(gstate);
892
893
  }
893
894
 
@@ -572,6 +572,15 @@ static unique_ptr<ComplexJSON> ParseJSON(const string &json, yyjson_doc *doc, yy
572
572
  const bool bool_val = yyjson_get_bool(root);
573
573
  return make_uniq<ComplexJSON>(bool_val ? "true" : "false");
574
574
  }
575
+ case YYJSON_TYPE_NUM | YYJSON_SUBTYPE_UINT:
576
+ return make_uniq<ComplexJSON>(to_string(unsafe_yyjson_get_uint(root)));
577
+ case YYJSON_TYPE_NUM | YYJSON_SUBTYPE_SINT:
578
+ return make_uniq<ComplexJSON>(to_string(unsafe_yyjson_get_sint(root)));
579
+ case YYJSON_TYPE_NUM | YYJSON_SUBTYPE_REAL:
580
+ case YYJSON_TYPE_RAW | YYJSON_SUBTYPE_NONE:
581
+ return make_uniq<ComplexJSON>(to_string(unsafe_yyjson_get_real(root)));
582
+ case YYJSON_TYPE_NULL | YYJSON_SUBTYPE_NONE:
583
+ return make_uniq<ComplexJSON>("null");
575
584
  default:
576
585
  yyjson_doc_free(doc);
577
586
  throw SerializationException("Failed to parse JSON string: %s", json);
@@ -693,6 +702,21 @@ string StringUtil::ToComplexJSONMap(const ComplexJSON &complex_json) {
693
702
  return ComplexJSON::GetValueRecursive(complex_json);
694
703
  }
695
704
 
705
+ string StringUtil::ValidateJSON(const char *data, const idx_t &len) {
706
+ // Same flags as in JSON extension
707
+ static constexpr auto READ_FLAG =
708
+ YYJSON_READ_ALLOW_INF_AND_NAN | YYJSON_READ_ALLOW_TRAILING_COMMAS | YYJSON_READ_BIGNUM_AS_RAW;
709
+ yyjson_read_err error;
710
+ yyjson_doc *doc = yyjson_read_opts((char *)data, len, READ_FLAG, nullptr, &error); // NOLINT: for yyjson
711
+ if (error.code != YYJSON_READ_SUCCESS) {
712
+ return StringUtil::Format("Malformed JSON at byte %lld of input: %s. Input: \"%s\"", error.pos, error.msg,
713
+ string(data, len));
714
+ }
715
+
716
+ yyjson_doc_free(doc);
717
+ return string();
718
+ }
719
+
696
720
  string StringUtil::ExceptionToJSONMap(ExceptionType type, const string &message,
697
721
  const unordered_map<string, string> &map) {
698
722
  D_ASSERT(map.find("exception_type") == map.end());
@@ -17,6 +17,7 @@ VirtualFileSystem::VirtualFileSystem(unique_ptr<FileSystem> &&inner) : default_f
17
17
 
18
18
  unique_ptr<FileHandle> VirtualFileSystem::OpenFileExtended(const OpenFileInfo &file, FileOpenFlags flags,
19
19
  optional_ptr<FileOpener> opener) {
20
+
20
21
  auto compression = flags.Compression();
21
22
  if (compression == FileCompressionType::AUTO_DETECT) {
22
23
  // auto-detect compression settings based on file name
@@ -34,8 +35,9 @@ unique_ptr<FileHandle> VirtualFileSystem::OpenFileExtended(const OpenFileInfo &f
34
35
  }
35
36
  }
36
37
  // open the base file handle in UNCOMPRESSED mode
38
+
37
39
  flags.SetCompression(FileCompressionType::UNCOMPRESSED);
38
- auto file_handle = FindFileSystem(file.path).OpenFile(file, flags, opener);
40
+ auto file_handle = FindFileSystem(file.path, opener).OpenFile(file, flags, opener);
39
41
  if (!file_handle) {
40
42
  return nullptr;
41
43
  }
@@ -111,7 +113,7 @@ void VirtualFileSystem::RemoveDirectory(const string &directory, optional_ptr<Fi
111
113
  bool VirtualFileSystem::ListFilesExtended(const string &directory,
112
114
  const std::function<void(OpenFileInfo &info)> &callback,
113
115
  optional_ptr<FileOpener> opener) {
114
- return FindFileSystem(directory).ListFiles(directory, callback, opener);
116
+ return FindFileSystem(directory, opener).ListFiles(directory, callback, opener);
115
117
  }
116
118
 
117
119
  void VirtualFileSystem::MoveFile(const string &source, const string &target, optional_ptr<FileOpener> opener) {
@@ -119,7 +121,7 @@ void VirtualFileSystem::MoveFile(const string &source, const string &target, opt
119
121
  }
120
122
 
121
123
  bool VirtualFileSystem::FileExists(const string &filename, optional_ptr<FileOpener> opener) {
122
- return FindFileSystem(filename).FileExists(filename, opener);
124
+ return FindFileSystem(filename, opener).FileExists(filename, opener);
123
125
  }
124
126
 
125
127
  bool VirtualFileSystem::IsPipe(const string &filename, optional_ptr<FileOpener> opener) {
@@ -139,7 +141,7 @@ string VirtualFileSystem::PathSeparator(const string &path) {
139
141
  }
140
142
 
141
143
  vector<OpenFileInfo> VirtualFileSystem::Glob(const string &path, FileOpener *opener) {
142
- return FindFileSystem(path).Glob(path, opener);
144
+ return FindFileSystem(path, opener).Glob(path, opener);
143
145
  }
144
146
 
145
147
  void VirtualFileSystem::RegisterSubSystem(unique_ptr<FileSystem> fs) {
@@ -216,16 +218,61 @@ bool VirtualFileSystem::SubSystemIsDisabled(const string &name) {
216
218
  return disabled_file_systems.find(name) != disabled_file_systems.end();
217
219
  }
218
220
 
221
+ FileSystem &VirtualFileSystem::FindFileSystem(const string &path, optional_ptr<FileOpener> opener) {
222
+ return FindFileSystem(path, FileOpener::TryGetDatabase(opener));
223
+ }
224
+
225
+ FileSystem &VirtualFileSystem::FindFileSystem(const string &path, optional_ptr<DatabaseInstance> db_instance) {
226
+ auto fs = FindFileSystemInternal(path);
227
+
228
+ if (!fs && db_instance) {
229
+ string required_extension;
230
+
231
+ for (const auto &entry : EXTENSION_FILE_PREFIXES) {
232
+ if (StringUtil::StartsWith(path, entry.name)) {
233
+ required_extension = entry.extension;
234
+ }
235
+ }
236
+ if (!required_extension.empty() && db_instance && !db_instance->ExtensionIsLoaded(required_extension)) {
237
+ auto &dbconfig = DBConfig::GetConfig(*db_instance);
238
+ if (!ExtensionHelper::CanAutoloadExtension(required_extension) ||
239
+ !dbconfig.options.autoload_known_extensions) {
240
+ auto error_message = "File " + path + " requires the extension " + required_extension + " to be loaded";
241
+ error_message =
242
+ ExtensionHelper::AddExtensionInstallHintToErrorMsg(*db_instance, error_message, required_extension);
243
+ throw MissingExtensionException(error_message);
244
+ }
245
+ // an extension is required to read this file, but it is not loaded - try to load it
246
+ ExtensionHelper::AutoLoadExtension(*db_instance, required_extension);
247
+ }
248
+
249
+ // Retry after having autoloaded
250
+ fs = FindFileSystem(path);
251
+ }
252
+
253
+ if (!fs) {
254
+ fs = default_fs;
255
+ }
256
+ if (!disabled_file_systems.empty() && disabled_file_systems.find(fs->GetName()) != disabled_file_systems.end()) {
257
+ throw PermissionException("File system %s has been disabled by configuration", fs->GetName());
258
+ }
259
+ return *fs;
260
+ }
261
+
219
262
  FileSystem &VirtualFileSystem::FindFileSystem(const string &path) {
220
- auto &fs = FindFileSystemInternal(path);
221
- if (!disabled_file_systems.empty() && disabled_file_systems.find(fs.GetName()) != disabled_file_systems.end()) {
222
- throw PermissionException("File system %s has been disabled by configuration", fs.GetName());
263
+ auto fs = FindFileSystemInternal(path);
264
+ if (!fs) {
265
+ fs = default_fs;
266
+ }
267
+ if (!disabled_file_systems.empty() && disabled_file_systems.find(fs->GetName()) != disabled_file_systems.end()) {
268
+ throw PermissionException("File system %s has been disabled by configuration", fs->GetName());
223
269
  }
224
- return fs;
270
+ return *fs;
225
271
  }
226
272
 
227
- FileSystem &VirtualFileSystem::FindFileSystemInternal(const string &path) {
273
+ optional_ptr<FileSystem> VirtualFileSystem::FindFileSystemInternal(const string &path) {
228
274
  FileSystem *fs = nullptr;
275
+
229
276
  for (auto &sub_system : sub_systems) {
230
277
  if (sub_system->CanHandleFile(path)) {
231
278
  if (sub_system->IsManuallySet()) {
@@ -237,7 +284,9 @@ FileSystem &VirtualFileSystem::FindFileSystemInternal(const string &path) {
237
284
  if (fs) {
238
285
  return *fs;
239
286
  }
240
- return *default_fs;
287
+
288
+ // We could use default_fs, that's on the caller
289
+ return nullptr;
241
290
  }
242
291
 
243
292
  } // namespace duckdb
@@ -217,9 +217,6 @@ void ARTMerger::MergeNodeAndPrefix(Node &node, Node &prefix, const GateStatus pa
217
217
  auto child = node.GetChildMutable(art, byte);
218
218
 
219
219
  // Reduce the prefix to the bytes after pos.
220
- // We always reduce by at least one byte,
221
- // thus, if the prefix was a gate, it no longer is.
222
- prefix.SetGateStatus(GateStatus::GATE_NOT_SET);
223
220
  Prefix::Reduce(art, prefix, pos);
224
221
 
225
222
  if (child) {
@@ -100,6 +100,10 @@ void Prefix::Reduce(ART &art, Node &node, const idx_t pos) {
100
100
  D_ASSERT(node.HasMetadata());
101
101
  D_ASSERT(pos < Count(art));
102
102
 
103
+ // We always reduce by at least one byte,
104
+ // thus, if the prefix was a gate, it no longer is.
105
+ node.SetGateStatus(GateStatus::GATE_NOT_SET);
106
+
103
107
  Prefix prefix(art, node);
104
108
  if (pos == idx_t(prefix.data[Count(art)] - 1)) {
105
109
  auto next = *prefix.ptr;
@@ -797,7 +797,7 @@ void StringValueResult::NullPaddingQuotedNewlineCheck() const {
797
797
  // If we have null_padding set, we found a quoted new line, we are scanning the file in parallel; We error.
798
798
  LinesPerBoundary lines_per_batch(iterator.GetBoundaryIdx(), lines_read);
799
799
  auto csv_error = CSVError::NullPaddingFail(state_machine.options, lines_per_batch, path);
800
- error_handler.Error(csv_error, try_row);
800
+ error_handler.Error(csv_error, true);
801
801
  }
802
802
  }
803
803
 
@@ -65,9 +65,9 @@ SourceResultType PhysicalReset::GetData(ExecutionContext &context, DataChunk &ch
65
65
  }
66
66
  if (variable_scope == SetScope::SESSION) {
67
67
  auto &client_config = ClientConfig::GetConfig(context.client);
68
- client_config.set_variables.erase(name.ToStdString());
68
+ client_config.set_variables.erase(option->name);
69
69
  } else {
70
- config.ResetGenericOption(name);
70
+ config.ResetGenericOption(option->name);
71
71
  }
72
72
  return SourceResultType::FINISHED;
73
73
  }
@@ -36,7 +36,7 @@ SourceResultType PhysicalAttach::GetData(ExecutionContext &context, DataChunk &c
36
36
  if (info->on_conflict == OnCreateConflict::IGNORE_ON_CONFLICT ||
37
37
  info->on_conflict == OnCreateConflict::REPLACE_ON_CONFLICT) {
38
38
  // constant-time lookup in the catalog for the db name
39
- auto existing_db = db_manager.GetDatabase(context.client, name);
39
+ auto existing_db = db_manager.GetDatabase(name);
40
40
  if (existing_db) {
41
41
  if ((existing_db->IsReadOnly() && options.access_mode == AccessMode::READ_WRITE) ||
42
42
  (!existing_db->IsReadOnly() && options.access_mode == AccessMode::READ_ONLY)) {
@@ -27,7 +27,7 @@ PhysicalPlanGenerator::PlanAsOfLoopJoin(LogicalComparisonJoin &op, PhysicalOpera
27
27
  //
28
28
  // ∏ * \ pk
29
29
  // |
30
- // Γ pk;first(P),arg_xxx(B,inequality)
30
+ // Γ pk;first(P),arg_xxx_null(B,inequality)
31
31
  // |
32
32
  // ∏ *,inequality
33
33
  // |
@@ -88,13 +88,13 @@ PhysicalPlanGenerator::PlanAsOfLoopJoin(LogicalComparisonJoin &op, PhysicalOpera
88
88
  case ExpressionType::COMPARE_GREATERTHAN:
89
89
  D_ASSERT(asof_idx == op.conditions.size());
90
90
  asof_idx = i;
91
- arg_min_max = "arg_max";
91
+ arg_min_max = "arg_max_null";
92
92
  break;
93
93
  case ExpressionType::COMPARE_LESSTHANOREQUALTO:
94
94
  case ExpressionType::COMPARE_LESSTHAN:
95
95
  D_ASSERT(asof_idx == op.conditions.size());
96
96
  asof_idx = i;
97
- arg_min_max = "arg_min";
97
+ arg_min_max = "arg_min_null";
98
98
  break;
99
99
  case ExpressionType::COMPARE_EQUAL:
100
100
  case ExpressionType::COMPARE_NOTEQUAL:
@@ -0,0 +1,45 @@
1
+ #include "duckdb/function/table/system_functions.hpp"
2
+
3
+ #include "duckdb/main/client_context.hpp"
4
+ #include "duckdb/main/database.hpp"
5
+ #include "duckdb/main/connection_manager.hpp"
6
+
7
+ namespace duckdb {
8
+
9
+ struct DuckDBConnectionCountData : public GlobalTableFunctionState {
10
+ DuckDBConnectionCountData() : count(0), finished(false) {
11
+ }
12
+ idx_t count;
13
+ bool finished;
14
+ };
15
+
16
+ static unique_ptr<FunctionData> DuckDBConnectionCountBind(ClientContext &context, TableFunctionBindInput &input,
17
+ vector<LogicalType> &return_types, vector<string> &names) {
18
+ names.emplace_back("count");
19
+ return_types.emplace_back(LogicalType::UBIGINT);
20
+ return nullptr;
21
+ }
22
+
23
+ unique_ptr<GlobalTableFunctionState> DuckDBConnectionCountInit(ClientContext &context, TableFunctionInitInput &input) {
24
+ auto result = make_uniq<DuckDBConnectionCountData>();
25
+ auto &conn_manager = context.db->GetConnectionManager();
26
+ result->count = conn_manager.GetConnectionCount();
27
+ return std::move(result);
28
+ }
29
+
30
+ void DuckDBConnectionCountFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) {
31
+ auto &data = data_p.global_state->Cast<DuckDBConnectionCountData>();
32
+ if (data.finished) {
33
+ return;
34
+ }
35
+ output.SetValue(0, 0, Value::UBIGINT(data.count));
36
+ output.SetCardinality(1);
37
+ data.finished = true;
38
+ }
39
+
40
+ void DuckDBConnectionCountFun::RegisterFunction(BuiltinFunctions &set) {
41
+ set.AddFunction(TableFunction("duckdb_connection_count", {}, DuckDBConnectionCountFunction,
42
+ DuckDBConnectionCountBind, DuckDBConnectionCountInit));
43
+ }
44
+
45
+ } // namespace duckdb
@@ -12,6 +12,10 @@ struct DuckDBSettingValue {
12
12
  string input_type;
13
13
  string scope;
14
14
  vector<Value> aliases;
15
+
16
+ inline bool operator<(const DuckDBSettingValue &rhs) const {
17
+ return name < rhs.name;
18
+ };
15
19
  };
16
20
 
17
21
  struct DuckDBSettingsData : public GlobalTableFunctionState {
@@ -79,7 +83,12 @@ unique_ptr<GlobalTableFunctionState> DuckDBSettingsInit(ClientContext &context,
79
83
  if (entry != aliases.end()) {
80
84
  value.aliases = std::move(entry->second);
81
85
  }
82
-
86
+ for (auto &alias : value.aliases) {
87
+ DuckDBSettingValue alias_value = value;
88
+ alias_value.name = StringValue::Get(alias);
89
+ alias_value.aliases.clear();
90
+ result->settings.push_back(std::move(alias_value));
91
+ }
83
92
  result->settings.push_back(std::move(value));
84
93
  }
85
94
  for (auto &ext_param : config.extension_parameters) {
@@ -98,6 +107,7 @@ unique_ptr<GlobalTableFunctionState> DuckDBSettingsInit(ClientContext &context,
98
107
 
99
108
  result->settings.push_back(std::move(value));
100
109
  }
110
+ std::sort(result->settings.begin(), result->settings.end());
101
111
  return std::move(result);
102
112
  }
103
113
 
@@ -18,6 +18,7 @@ void BuiltinFunctions::RegisterSQLiteFunctions() {
18
18
  PragmaDatabaseSize::RegisterFunction(*this);
19
19
  PragmaUserAgent::RegisterFunction(*this);
20
20
 
21
+ DuckDBConnectionCountFun::RegisterFunction(*this);
21
22
  DuckDBApproxDatabaseCountFun::RegisterFunction(*this);
22
23
  DuckDBColumnsFun::RegisterFunction(*this);
23
24
  DuckDBConstraintsFun::RegisterFunction(*this);
@@ -1,5 +1,5 @@
1
1
  #ifndef DUCKDB_PATCH_VERSION
2
- #define DUCKDB_PATCH_VERSION "0"
2
+ #define DUCKDB_PATCH_VERSION "1"
3
3
  #endif
4
4
  #ifndef DUCKDB_MINOR_VERSION
5
5
  #define DUCKDB_MINOR_VERSION 4
@@ -8,10 +8,10 @@
8
8
  #define DUCKDB_MAJOR_VERSION 1
9
9
  #endif
10
10
  #ifndef DUCKDB_VERSION
11
- #define DUCKDB_VERSION "v1.4.0"
11
+ #define DUCKDB_VERSION "v1.4.1"
12
12
  #endif
13
13
  #ifndef DUCKDB_SOURCE_ID
14
- #define DUCKDB_SOURCE_ID "b8a06e4a22"
14
+ #define DUCKDB_SOURCE_ID "b390a7c376"
15
15
  #endif
16
16
  #include "duckdb/function/table/system_functions.hpp"
17
17
  #include "duckdb/main/database.hpp"
@@ -234,6 +234,8 @@ enum class LogLevel : uint8_t;
234
234
 
235
235
  enum class LogMode : uint8_t;
236
236
 
237
+ enum class LoggingTargetTable : uint8_t;
238
+
237
239
  enum class LogicalOperatorType : uint8_t;
238
240
 
239
241
  enum class LogicalTypeId : uint8_t;
@@ -748,6 +750,9 @@ const char* EnumUtil::ToChars<LogLevel>(LogLevel value);
748
750
  template<>
749
751
  const char* EnumUtil::ToChars<LogMode>(LogMode value);
750
752
 
753
+ template<>
754
+ const char* EnumUtil::ToChars<LoggingTargetTable>(LoggingTargetTable value);
755
+
751
756
  template<>
752
757
  const char* EnumUtil::ToChars<LogicalOperatorType>(LogicalOperatorType value);
753
758
 
@@ -1367,6 +1372,9 @@ LogLevel EnumUtil::FromString<LogLevel>(const char *value);
1367
1372
  template<>
1368
1373
  LogMode EnumUtil::FromString<LogMode>(const char *value);
1369
1374
 
1375
+ template<>
1376
+ LoggingTargetTable EnumUtil::FromString<LoggingTargetTable>(const char *value);
1377
+
1370
1378
  template<>
1371
1379
  LogicalOperatorType EnumUtil::FromString<LogicalOperatorType>(const char *value);
1372
1380