duckdb 1.4.2-dev4.0 → 1.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. package/package.json +2 -2
  2. package/src/duckdb/extension/icu/icu_extension.cpp +67 -6
  3. package/src/duckdb/extension/icu/third_party/icu/common/putil.cpp +9 -3
  4. package/src/duckdb/extension/json/include/json_serializer.hpp +12 -0
  5. package/src/duckdb/extension/json/json_functions/json_create.cpp +10 -10
  6. package/src/duckdb/extension/parquet/decoder/delta_length_byte_array_decoder.cpp +19 -5
  7. package/src/duckdb/extension/parquet/include/decoder/delta_length_byte_array_decoder.hpp +1 -1
  8. package/src/duckdb/extension/parquet/include/parquet_dbp_decoder.hpp +11 -2
  9. package/src/duckdb/extension/parquet/include/reader/string_column_reader.hpp +2 -1
  10. package/src/duckdb/extension/parquet/parquet_reader.cpp +3 -1
  11. package/src/duckdb/extension/parquet/parquet_writer.cpp +16 -1
  12. package/src/duckdb/extension/parquet/reader/string_column_reader.cpp +1 -1
  13. package/src/duckdb/extension/parquet/writer/primitive_column_writer.cpp +1 -1
  14. package/src/duckdb/src/catalog/default/default_table_functions.cpp +1 -1
  15. package/src/duckdb/src/common/adbc/adbc.cpp +8 -6
  16. package/src/duckdb/src/common/csv_writer.cpp +1 -13
  17. package/src/duckdb/src/common/encryption_key_manager.cpp +10 -9
  18. package/src/duckdb/src/common/enum_util.cpp +19 -0
  19. package/src/duckdb/src/common/enums/compression_type.cpp +51 -16
  20. package/src/duckdb/src/common/exception/binder_exception.cpp +7 -2
  21. package/src/duckdb/src/common/progress_bar/unscented_kalman_filter.cpp +2 -2
  22. package/src/duckdb/src/common/random_engine.cpp +10 -0
  23. package/src/duckdb/src/execution/expression_executor/execute_comparison.cpp +13 -2
  24. package/src/duckdb/src/execution/index/art/art.cpp +6 -3
  25. package/src/duckdb/src/execution/index/bound_index.cpp +32 -21
  26. package/src/duckdb/src/execution/index/unbound_index.cpp +20 -9
  27. package/src/duckdb/src/execution/join_hashtable.cpp +9 -3
  28. package/src/duckdb/src/execution/operator/helper/physical_buffered_batch_collector.cpp +1 -1
  29. package/src/duckdb/src/execution/operator/helper/physical_buffered_collector.cpp +1 -1
  30. package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp +5 -0
  31. package/src/duckdb/src/function/cast/cast_function_set.cpp +3 -1
  32. package/src/duckdb/src/function/macro_function.cpp +1 -1
  33. package/src/duckdb/src/function/scalar/compressed_materialization/compress_string.cpp +1 -1
  34. package/src/duckdb/src/function/scalar/create_sort_key.cpp +5 -3
  35. package/src/duckdb/src/function/scalar/operator/arithmetic.cpp +1 -1
  36. package/src/duckdb/src/function/scalar/system/parse_log_message.cpp +4 -2
  37. package/src/duckdb/src/function/table/copy_csv.cpp +28 -4
  38. package/src/duckdb/src/function/table/direct_file_reader.cpp +10 -0
  39. package/src/duckdb/src/function/table/read_file.cpp +65 -1
  40. package/src/duckdb/src/function/table/version/pragma_version.cpp +3 -3
  41. package/src/duckdb/src/include/duckdb/common/csv_writer.hpp +0 -3
  42. package/src/duckdb/src/include/duckdb/common/encryption_key_manager.hpp +2 -0
  43. package/src/duckdb/src/include/duckdb/common/encryption_state.hpp +5 -0
  44. package/src/duckdb/src/include/duckdb/common/enum_util.hpp +8 -0
  45. package/src/duckdb/src/include/duckdb/common/enums/compression_type.hpp +42 -2
  46. package/src/duckdb/src/include/duckdb/common/http_util.hpp +7 -0
  47. package/src/duckdb/src/include/duckdb/common/hugeint.hpp +1 -1
  48. package/src/duckdb/src/include/duckdb/common/operator/comparison_operators.hpp +0 -11
  49. package/src/duckdb/src/include/duckdb/common/random_engine.hpp +2 -0
  50. package/src/duckdb/src/include/duckdb/common/sort/duckdb_pdqsort.hpp +1 -0
  51. package/src/duckdb/src/include/duckdb/common/types/hugeint.hpp +6 -6
  52. package/src/duckdb/src/include/duckdb/common/types/row/block_iterator.hpp +115 -97
  53. package/src/duckdb/src/include/duckdb/execution/index/art/art_operator.hpp +54 -0
  54. package/src/duckdb/src/include/duckdb/execution/index/bound_index.hpp +21 -2
  55. package/src/duckdb/src/include/duckdb/execution/index/unbound_index.hpp +26 -8
  56. package/src/duckdb/src/include/duckdb/execution/join_hashtable.hpp +2 -0
  57. package/src/duckdb/src/include/duckdb/function/table/read_file.hpp +0 -49
  58. package/src/duckdb/src/include/duckdb/logging/log_manager.hpp +1 -1
  59. package/src/duckdb/src/include/duckdb/logging/log_type.hpp +14 -0
  60. package/src/duckdb/src/include/duckdb/main/attached_database.hpp +2 -1
  61. package/src/duckdb/src/include/duckdb/main/buffered_data/batched_buffered_data.hpp +1 -1
  62. package/src/duckdb/src/include/duckdb/main/buffered_data/buffered_data.hpp +1 -1
  63. package/src/duckdb/src/include/duckdb/main/buffered_data/simple_buffered_data.hpp +1 -1
  64. package/src/duckdb/src/include/duckdb/main/capi/capi_internal.hpp +2 -0
  65. package/src/duckdb/src/include/duckdb/main/database.hpp +2 -2
  66. package/src/duckdb/src/include/duckdb/main/database_file_path_manager.hpp +10 -6
  67. package/src/duckdb/src/include/duckdb/main/extension_entries.hpp +4 -0
  68. package/src/duckdb/src/include/duckdb/main/profiling_info.hpp +1 -0
  69. package/src/duckdb/src/include/duckdb/main/query_profiler.hpp +1 -0
  70. package/src/duckdb/src/include/duckdb/main/relation/create_table_relation.hpp +3 -0
  71. package/src/duckdb/src/include/duckdb/main/relation/insert_relation.hpp +2 -0
  72. package/src/duckdb/src/include/duckdb/main/relation/table_relation.hpp +2 -0
  73. package/src/duckdb/src/include/duckdb/main/relation.hpp +10 -2
  74. package/src/duckdb/src/include/duckdb/main/settings.hpp +9 -0
  75. package/src/duckdb/src/include/duckdb/optimizer/filter_pullup.hpp +10 -14
  76. package/src/duckdb/src/include/duckdb/optimizer/join_order/relation_manager.hpp +5 -1
  77. package/src/duckdb/src/include/duckdb/parser/query_node.hpp +3 -0
  78. package/src/duckdb/src/include/duckdb/planner/bound_statement.hpp +1 -0
  79. package/src/duckdb/src/include/duckdb/storage/block.hpp +9 -0
  80. package/src/duckdb/src/include/duckdb/storage/block_manager.hpp +9 -2
  81. package/src/duckdb/src/include/duckdb/storage/index.hpp +8 -2
  82. package/src/duckdb/src/include/duckdb/storage/metadata/metadata_manager.hpp +2 -0
  83. package/src/duckdb/src/include/duckdb/storage/metadata/metadata_reader.hpp +1 -1
  84. package/src/duckdb/src/include/duckdb/storage/storage_options.hpp +0 -7
  85. package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +6 -2
  86. package/src/duckdb/src/include/duckdb/verification/deserialized_statement_verifier.hpp +6 -0
  87. package/src/duckdb/src/logging/log_manager.cpp +2 -1
  88. package/src/duckdb/src/logging/log_types.cpp +30 -1
  89. package/src/duckdb/src/main/attached_database.cpp +4 -7
  90. package/src/duckdb/src/main/buffered_data/batched_buffered_data.cpp +2 -3
  91. package/src/duckdb/src/main/buffered_data/buffered_data.cpp +2 -3
  92. package/src/duckdb/src/main/buffered_data/simple_buffered_data.cpp +1 -2
  93. package/src/duckdb/src/main/capi/prepared-c.cpp +9 -2
  94. package/src/duckdb/src/main/config.cpp +6 -5
  95. package/src/duckdb/src/main/database.cpp +9 -3
  96. package/src/duckdb/src/main/database_file_path_manager.cpp +43 -14
  97. package/src/duckdb/src/main/database_manager.cpp +1 -1
  98. package/src/duckdb/src/main/http/http_util.cpp +19 -1
  99. package/src/duckdb/src/main/profiling_info.cpp +11 -0
  100. package/src/duckdb/src/main/query_profiler.cpp +16 -0
  101. package/src/duckdb/src/main/relation/create_table_relation.cpp +9 -0
  102. package/src/duckdb/src/main/relation/insert_relation.cpp +7 -0
  103. package/src/duckdb/src/main/relation/table_relation.cpp +14 -0
  104. package/src/duckdb/src/main/relation.cpp +28 -12
  105. package/src/duckdb/src/main/settings/custom_settings.cpp +9 -3
  106. package/src/duckdb/src/optimizer/filter_pullup.cpp +14 -0
  107. package/src/duckdb/src/optimizer/join_order/relation_manager.cpp +29 -10
  108. package/src/duckdb/src/optimizer/rule/regex_optimizations.cpp +7 -0
  109. package/src/duckdb/src/parallel/task_executor.cpp +4 -2
  110. package/src/duckdb/src/parser/query_node/cte_node.cpp +79 -0
  111. package/src/duckdb/src/parser/transform/expression/transform_cast.cpp +3 -1
  112. package/src/duckdb/src/planner/binder/expression/bind_macro_expression.cpp +1 -0
  113. package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +12 -4
  114. package/src/duckdb/src/planner/binder/statement/bind_insert.cpp +16 -12
  115. package/src/duckdb/src/planner/binder/statement/bind_merge_into.cpp +42 -5
  116. package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +0 -24
  117. package/src/duckdb/src/planner/binder/tableref/bind_table_function.cpp +1 -1
  118. package/src/duckdb/src/planner/binder.cpp +0 -1
  119. package/src/duckdb/src/planner/expression_binder/having_binder.cpp +1 -2
  120. package/src/duckdb/src/storage/buffer/block_manager.cpp +20 -6
  121. package/src/duckdb/src/storage/checkpoint/table_data_writer.cpp +8 -6
  122. package/src/duckdb/src/storage/checkpoint_manager.cpp +24 -22
  123. package/src/duckdb/src/storage/compression/validity_uncompressed.cpp +7 -0
  124. package/src/duckdb/src/storage/compression/zstd.cpp +34 -12
  125. package/src/duckdb/src/storage/data_table.cpp +1 -1
  126. package/src/duckdb/src/storage/local_storage.cpp +15 -2
  127. package/src/duckdb/src/storage/metadata/metadata_manager.cpp +29 -6
  128. package/src/duckdb/src/storage/metadata/metadata_reader.cpp +11 -15
  129. package/src/duckdb/src/storage/metadata/metadata_writer.cpp +1 -1
  130. package/src/duckdb/src/storage/serialization/serialize_query_node.cpp +1 -19
  131. package/src/duckdb/src/storage/single_file_block_manager.cpp +33 -3
  132. package/src/duckdb/src/storage/standard_buffer_manager.cpp +3 -1
  133. package/src/duckdb/src/storage/storage_info.cpp +4 -0
  134. package/src/duckdb/src/storage/storage_manager.cpp +8 -0
  135. package/src/duckdb/src/storage/table/array_column_data.cpp +1 -1
  136. package/src/duckdb/src/storage/table/column_data.cpp +3 -2
  137. package/src/duckdb/src/storage/table/column_data_checkpointer.cpp +3 -2
  138. package/src/duckdb/src/storage/table/row_group.cpp +41 -24
  139. package/src/duckdb/src/storage/table/row_group_collection.cpp +114 -11
  140. package/src/duckdb/src/storage/table_index_list.cpp +18 -5
  141. package/src/duckdb/src/transaction/cleanup_state.cpp +7 -2
  142. package/src/duckdb/third_party/mbedtls/include/mbedtls_wrapper.hpp +5 -0
  143. package/src/duckdb/third_party/mbedtls/mbedtls_wrapper.cpp +8 -21
  144. package/src/duckdb/third_party/parquet/parquet_types.cpp +57 -35
  145. package/src/duckdb/third_party/parquet/parquet_types.h +9 -2
  146. package/src/duckdb/ub_src_common_types_row.cpp +0 -2
@@ -205,7 +205,7 @@ void LogManager::SetEnableStructuredLoggers(vector<string> &enabled_logger_types
205
205
  throw InvalidInputException("Unknown log type: '%s'", enabled_logger_type);
206
206
  }
207
207
 
208
- new_config.enabled_log_types.insert(enabled_logger_type);
208
+ new_config.enabled_log_types.insert(lookup->name);
209
209
 
210
210
  min_log_level = MinValue(min_log_level, lookup->level);
211
211
  }
@@ -266,6 +266,7 @@ void LogManager::RegisterDefaultLogTypes() {
266
266
  RegisterLogType(make_uniq<HTTPLogType>());
267
267
  RegisterLogType(make_uniq<QueryLogType>());
268
268
  RegisterLogType(make_uniq<PhysicalOperatorLogType>());
269
+ RegisterLogType(make_uniq<MetricsLogType>());
269
270
  }
270
271
 
271
272
  } // namespace duckdb
@@ -14,6 +14,7 @@ constexpr LogLevel FileSystemLogType::LEVEL;
14
14
  constexpr LogLevel QueryLogType::LEVEL;
15
15
  constexpr LogLevel HTTPLogType::LEVEL;
16
16
  constexpr LogLevel PhysicalOperatorLogType::LEVEL;
17
+ constexpr LogLevel MetricsLogType::LEVEL;
17
18
  constexpr LogLevel CheckpointLogType::LEVEL;
18
19
 
19
20
  //===--------------------------------------------------------------------===//
@@ -58,6 +59,8 @@ LogicalType HTTPLogType::GetLogType() {
58
59
  child_list_t<LogicalType> request_child_list = {
59
60
  {"type", LogicalType::VARCHAR},
60
61
  {"url", LogicalType::VARCHAR},
62
+ {"start_time", LogicalType::TIMESTAMP_TZ},
63
+ {"duration_ms", LogicalType::BIGINT},
61
64
  {"headers", LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR)},
62
65
  };
63
66
  auto request_type = LogicalType::STRUCT(request_child_list);
@@ -90,7 +93,10 @@ string HTTPLogType::ConstructLogMessage(BaseRequest &request, optional_ptr<HTTPR
90
93
  {"type", Value(EnumUtil::ToString(request.type))},
91
94
  {"url", Value(request.url)},
92
95
  {"headers", CreateHTTPHeadersValue(request.headers)},
93
- };
96
+ {"start_time", request.have_request_timing ? Value::TIMESTAMP(request.request_start) : Value()},
97
+ {"duration_ms", request.have_request_timing ? Value::BIGINT(Timestamp::GetEpochMs(request.request_end) -
98
+ Timestamp::GetEpochMs(request.request_start))
99
+ : Value()}};
94
100
  auto request_value = Value::STRUCT(request_child_list);
95
101
  Value response_value;
96
102
  if (response) {
@@ -147,6 +153,29 @@ string PhysicalOperatorLogType::ConstructLogMessage(const PhysicalOperator &phys
147
153
 
148
154
  return Value::STRUCT(std::move(child_list)).ToString();
149
155
  }
156
+
157
+ //===--------------------------------------------------------------------===//
158
+ // MetricsLogType
159
+ //===--------------------------------------------------------------------===//
160
+ MetricsLogType::MetricsLogType() : LogType(NAME, LEVEL, GetLogType()) {
161
+ }
162
+
163
+ LogicalType MetricsLogType::GetLogType() {
164
+ child_list_t<LogicalType> child_list = {
165
+ {"metric", LogicalType::VARCHAR},
166
+ {"value", LogicalType::VARCHAR},
167
+ };
168
+ return LogicalType::STRUCT(child_list);
169
+ }
170
+
171
+ string MetricsLogType::ConstructLogMessage(const MetricsType &metric, const Value &value) {
172
+ child_list_t<Value> child_list = {
173
+ {"metric", EnumUtil::ToString(metric)},
174
+ {"value", value.ToString()},
175
+ };
176
+ return Value::STRUCT(std::move(child_list)).ToString();
177
+ }
178
+
150
179
  //===--------------------------------------------------------------------===//
151
180
  // CheckpointLogType
152
181
  //===--------------------------------------------------------------------===//
@@ -14,8 +14,9 @@
14
14
 
15
15
  namespace duckdb {
16
16
 
17
- StoredDatabasePath::StoredDatabasePath(DatabaseFilePathManager &manager, string path_p, const string &name)
18
- : manager(manager), path(std::move(path_p)) {
17
+ StoredDatabasePath::StoredDatabasePath(DatabaseManager &db_manager, DatabaseFilePathManager &manager, string path_p,
18
+ const string &name)
19
+ : db_manager(db_manager), manager(manager), path(std::move(path_p)) {
19
20
  }
20
21
 
21
22
  StoredDatabasePath::~StoredDatabasePath() {
@@ -23,7 +24,7 @@ StoredDatabasePath::~StoredDatabasePath() {
23
24
  }
24
25
 
25
26
  void StoredDatabasePath::OnDetach() {
26
- manager.DetachDatabase(path);
27
+ manager.DetachDatabase(db_manager, path);
27
28
  }
28
29
 
29
30
  //===--------------------------------------------------------------------===//
@@ -271,10 +272,6 @@ void AttachedDatabase::Close() {
271
272
  catalog.reset();
272
273
  storage.reset();
273
274
  stored_database_path.reset();
274
-
275
- if (Allocator::SupportsFlush()) {
276
- Allocator::FlushAll();
277
- }
278
275
  }
279
276
 
280
277
  } // namespace duckdb
@@ -14,9 +14,8 @@ void BatchedBufferedData::BlockSink(const InterruptState &blocked_sink, idx_t ba
14
14
  blocked_sinks.emplace(batch, blocked_sink);
15
15
  }
16
16
 
17
- BatchedBufferedData::BatchedBufferedData(weak_ptr<ClientContext> context)
18
- : BufferedData(BufferedData::Type::BATCHED, std::move(context)), buffer_byte_count(0), read_queue_byte_count(0),
19
- min_batch(0) {
17
+ BatchedBufferedData::BatchedBufferedData(ClientContext &context)
18
+ : BufferedData(BufferedData::Type::BATCHED, context), buffer_byte_count(0), read_queue_byte_count(0), min_batch(0) {
20
19
  read_queue_capacity = (idx_t)(static_cast<double>(total_buffer_size) * 0.6);
21
20
  buffer_capacity = (idx_t)(static_cast<double>(total_buffer_size) * 0.4);
22
21
  }
@@ -4,9 +4,8 @@
4
4
 
5
5
  namespace duckdb {
6
6
 
7
- BufferedData::BufferedData(Type type, weak_ptr<ClientContext> context_p) : type(type), context(std::move(context_p)) {
8
- auto client_context = context.lock();
9
- auto &config = ClientConfig::GetConfig(*client_context);
7
+ BufferedData::BufferedData(Type type, ClientContext &context_p) : type(type), context(context_p.shared_from_this()) {
8
+ auto &config = ClientConfig::GetConfig(context_p);
10
9
  total_buffer_size = config.streaming_buffer_size;
11
10
  }
12
11
 
@@ -6,8 +6,7 @@
6
6
 
7
7
  namespace duckdb {
8
8
 
9
- SimpleBufferedData::SimpleBufferedData(weak_ptr<ClientContext> context)
10
- : BufferedData(BufferedData::Type::SIMPLE, std::move(context)) {
9
+ SimpleBufferedData::SimpleBufferedData(ClientContext &context) : BufferedData(BufferedData::Type::SIMPLE, context) {
11
10
  buffered_count = 0;
12
11
  buffer_size = total_buffer_size;
13
12
  }
@@ -88,7 +88,13 @@ duckdb_state duckdb_prepare(duckdb_connection connection, const char *query,
88
88
 
89
89
  const char *duckdb_prepare_error(duckdb_prepared_statement prepared_statement) {
90
90
  auto wrapper = reinterpret_cast<PreparedStatementWrapper *>(prepared_statement);
91
- if (!wrapper || !wrapper->statement || !wrapper->statement->HasError()) {
91
+ if (!wrapper) {
92
+ return nullptr;
93
+ }
94
+ if (!wrapper->success) {
95
+ return wrapper->error_data.Message().c_str();
96
+ }
97
+ if (!wrapper->statement || !wrapper->statement->HasError()) {
92
98
  return nullptr;
93
99
  }
94
100
  return wrapper->statement->error.Message().c_str();
@@ -229,9 +235,10 @@ duckdb_state duckdb_bind_value(duckdb_prepared_statement prepared_statement, idx
229
235
  return DuckDBError;
230
236
  }
231
237
  if (param_idx <= 0 || param_idx > wrapper->statement->named_param_map.size()) {
232
- wrapper->statement->error =
238
+ wrapper->error_data =
233
239
  duckdb::InvalidInputException("Can not bind to parameter number %d, statement only has %d parameter(s)",
234
240
  param_idx, wrapper->statement->named_param_map.size());
241
+ wrapper->success = false;
235
242
  return DuckDBError;
236
243
  }
237
244
  auto identifier = duckdb_parameter_name_internal(prepared_statement, param_idx);
@@ -86,6 +86,7 @@ static const ConfigurationOption internal_options[] = {
86
86
  DUCKDB_LOCAL(DebugForceExternalSetting),
87
87
  DUCKDB_SETTING(DebugForceNoCrossProductSetting),
88
88
  DUCKDB_SETTING(DebugSkipCheckpointOnCommitSetting),
89
+ DUCKDB_SETTING(DebugVerifyBlocksSetting),
89
90
  DUCKDB_SETTING_CALLBACK(DebugVerifyVectorSetting),
90
91
  DUCKDB_SETTING_CALLBACK(DebugWindowModeSetting),
91
92
  DUCKDB_GLOBAL(DefaultBlockSizeSetting),
@@ -177,12 +178,12 @@ static const ConfigurationOption internal_options[] = {
177
178
  DUCKDB_GLOBAL(ZstdMinStringLengthSetting),
178
179
  FINAL_SETTING};
179
180
 
180
- static const ConfigurationAlias setting_aliases[] = {DUCKDB_SETTING_ALIAS("memory_limit", 83),
181
- DUCKDB_SETTING_ALIAS("null_order", 33),
182
- DUCKDB_SETTING_ALIAS("profiling_output", 102),
183
- DUCKDB_SETTING_ALIAS("user", 116),
181
+ static const ConfigurationAlias setting_aliases[] = {DUCKDB_SETTING_ALIAS("memory_limit", 84),
182
+ DUCKDB_SETTING_ALIAS("null_order", 34),
183
+ DUCKDB_SETTING_ALIAS("profiling_output", 103),
184
+ DUCKDB_SETTING_ALIAS("user", 117),
184
185
  DUCKDB_SETTING_ALIAS("wal_autocheckpoint", 20),
185
- DUCKDB_SETTING_ALIAS("worker_threads", 115),
186
+ DUCKDB_SETTING_ALIAS("worker_threads", 116),
186
187
  FINAL_ALIAS};
187
188
 
188
189
  vector<ConfigurationOption> DBConfig::GetOptions() {
@@ -285,7 +285,7 @@ void DatabaseInstance::Initialize(const char *database_path, DBConfig *user_conf
285
285
  buffer_manager = make_uniq<StandardBufferManager>(*this, config.options.temporary_directory);
286
286
  }
287
287
 
288
- log_manager = make_shared_ptr<LogManager>(*this, LogConfig());
288
+ log_manager = make_uniq<LogManager>(*this, LogConfig());
289
289
  log_manager->Initialize();
290
290
 
291
291
  external_file_cache = make_uniq<ExternalFileCache>(*this, config.options.enable_external_file_cache);
@@ -507,12 +507,18 @@ SettingLookupResult DatabaseInstance::TryGetCurrentSetting(const string &key, Va
507
507
  return db_config.TryGetCurrentSetting(key, result);
508
508
  }
509
509
 
510
- shared_ptr<EncryptionUtil> DatabaseInstance::GetEncryptionUtil() const {
510
+ shared_ptr<EncryptionUtil> DatabaseInstance::GetEncryptionUtil() {
511
+ if (!config.encryption_util || !config.encryption_util->SupportsEncryption()) {
512
+ ExtensionHelper::TryAutoLoadExtension(*this, "httpfs");
513
+ }
514
+
511
515
  if (config.encryption_util) {
512
516
  return config.encryption_util;
513
517
  }
514
518
 
515
- return make_shared_ptr<duckdb_mbedtls::MbedTlsWrapper::AESStateMBEDTLSFactory>();
519
+ auto result = make_shared_ptr<duckdb_mbedtls::MbedTlsWrapper::AESStateMBEDTLSFactory>();
520
+
521
+ return std::move(result);
516
522
  }
517
523
 
518
524
  ValidChecker &DatabaseInstance::GetValidChecker() {
@@ -5,35 +5,57 @@
5
5
 
6
6
  namespace duckdb {
7
7
 
8
+ DatabasePathInfo::DatabasePathInfo(DatabaseManager &manager, string name_p, AccessMode access_mode)
9
+ : name(std::move(name_p)), access_mode(access_mode) {
10
+ attached_databases.insert(manager);
11
+ }
12
+
8
13
  idx_t DatabaseFilePathManager::ApproxDatabaseCount() const {
9
14
  lock_guard<mutex> path_lock(db_paths_lock);
10
15
  return db_paths.size();
11
16
  }
12
17
 
13
- InsertDatabasePathResult DatabaseFilePathManager::InsertDatabasePath(const string &path, const string &name,
14
- OnCreateConflict on_conflict,
18
+ InsertDatabasePathResult DatabaseFilePathManager::InsertDatabasePath(DatabaseManager &manager, const string &path,
19
+ const string &name, OnCreateConflict on_conflict,
15
20
  AttachOptions &options) {
16
21
  if (path.empty() || path == IN_MEMORY_PATH) {
17
22
  return InsertDatabasePathResult::SUCCESS;
18
23
  }
19
24
 
20
25
  lock_guard<mutex> path_lock(db_paths_lock);
21
- auto entry = db_paths.emplace(path, DatabasePathInfo(name));
26
+ auto entry = db_paths.emplace(path, DatabasePathInfo(manager, name, options.access_mode));
22
27
  if (!entry.second) {
23
28
  auto &existing = entry.first->second;
29
+ bool already_exists = false;
30
+ bool attached_in_this_system = false;
24
31
  if (on_conflict == OnCreateConflict::IGNORE_ON_CONFLICT && existing.name == name) {
25
- if (existing.is_attached) {
32
+ already_exists = true;
33
+ attached_in_this_system = existing.attached_databases.find(manager) != existing.attached_databases.end();
34
+ }
35
+ if (options.access_mode == AccessMode::READ_ONLY && existing.access_mode == AccessMode::READ_ONLY) {
36
+ if (attached_in_this_system) {
26
37
  return InsertDatabasePathResult::ALREADY_EXISTS;
27
38
  }
28
- throw BinderException("Unique file handle conflict: Cannot attach \"%s\" - the database file \"%s\" is in "
29
- "the process of being detached",
30
- name, path);
39
+ // all attaches are in read-only mode - there is no conflict, just increase the reference count
40
+ existing.attached_databases.insert(manager);
41
+ existing.reference_count++;
42
+ } else {
43
+ if (already_exists) {
44
+ if (attached_in_this_system) {
45
+ return InsertDatabasePathResult::ALREADY_EXISTS;
46
+ }
47
+ throw BinderException(
48
+ "Unique file handle conflict: Cannot attach \"%s\" - the database file \"%s\" is in "
49
+ "the process of being detached",
50
+ name, path);
51
+ }
52
+ throw BinderException(
53
+ "Unique file handle conflict: Cannot attach \"%s\" - the database file \"%s\" is already "
54
+ "attached by database \"%s\"",
55
+ name, path, existing.name);
31
56
  }
32
- throw BinderException("Unique file handle conflict: Cannot attach \"%s\" - the database file \"%s\" is already "
33
- "attached by database \"%s\"",
34
- name, path, existing.name);
35
57
  }
36
- options.stored_database_path = make_uniq<StoredDatabasePath>(*this, path, name);
58
+ options.stored_database_path = make_uniq<StoredDatabasePath>(manager, *this, path, name);
37
59
  return InsertDatabasePathResult::SUCCESS;
38
60
  }
39
61
 
@@ -42,17 +64,24 @@ void DatabaseFilePathManager::EraseDatabasePath(const string &path) {
42
64
  return;
43
65
  }
44
66
  lock_guard<mutex> path_lock(db_paths_lock);
45
- db_paths.erase(path);
67
+ auto entry = db_paths.find(path);
68
+ if (entry != db_paths.end()) {
69
+ if (entry->second.reference_count <= 1) {
70
+ db_paths.erase(entry);
71
+ } else {
72
+ entry->second.reference_count--;
73
+ }
74
+ }
46
75
  }
47
76
 
48
- void DatabaseFilePathManager::DetachDatabase(const string &path) {
77
+ void DatabaseFilePathManager::DetachDatabase(DatabaseManager &manager, const string &path) {
49
78
  if (path.empty() || path == IN_MEMORY_PATH) {
50
79
  return;
51
80
  }
52
81
  lock_guard<mutex> path_lock(db_paths_lock);
53
82
  auto entry = db_paths.find(path);
54
83
  if (entry != db_paths.end()) {
55
- entry->second.is_attached = false;
84
+ entry->second.attached_databases.erase(manager);
56
85
  }
57
86
  }
58
87
 
@@ -199,7 +199,7 @@ idx_t DatabaseManager::ApproxDatabaseCount() {
199
199
  }
200
200
 
201
201
  InsertDatabasePathResult DatabaseManager::InsertDatabasePath(const AttachInfo &info, AttachOptions &options) {
202
- return path_manager->InsertDatabasePath(info.path, info.name, info.on_conflict, options);
202
+ return path_manager->InsertDatabasePath(*this, info.path, info.name, info.on_conflict, options);
203
203
  }
204
204
 
205
205
  vector<string> DatabaseManager::GetAttachedDatabasePaths() {
@@ -130,9 +130,12 @@ BaseRequest::BaseRequest(RequestType type, const string &url, const HTTPHeaders
130
130
  class HTTPLibClient : public HTTPClient {
131
131
  public:
132
132
  HTTPLibClient(HTTPParams &http_params, const string &proto_host_port) {
133
+ client = make_uniq<duckdb_httplib::Client>(proto_host_port);
134
+ Initialize(http_params);
135
+ }
136
+ void Initialize(HTTPParams &http_params) override {
133
137
  auto sec = static_cast<time_t>(http_params.timeout);
134
138
  auto usec = static_cast<time_t>(http_params.timeout_usec);
135
- client = make_uniq<duckdb_httplib::Client>(proto_host_port);
136
139
  client->set_follow_location(http_params.follow_location);
137
140
  client->set_keep_alive(http_params.keep_alive);
138
141
  client->set_write_timeout(sec, usec);
@@ -228,12 +231,27 @@ unique_ptr<HTTPResponse> HTTPUtil::SendRequest(BaseRequest &request, unique_ptr<
228
231
 
229
232
  std::function<unique_ptr<HTTPResponse>(void)> on_request([&]() {
230
233
  unique_ptr<HTTPResponse> response;
234
+
235
+ // When logging is enabled, we collect request timings
236
+ if (request.params.logger) {
237
+ request.have_request_timing = request.params.logger->ShouldLog(HTTPLogType::NAME, HTTPLogType::LEVEL);
238
+ }
239
+
231
240
  try {
241
+ if (request.have_request_timing) {
242
+ request.request_start = Timestamp::GetCurrentTimestamp();
243
+ }
232
244
  response = client->Request(request);
233
245
  } catch (...) {
246
+ if (request.have_request_timing) {
247
+ request.request_end = Timestamp::GetCurrentTimestamp();
248
+ }
234
249
  LogRequest(request, nullptr);
235
250
  throw;
236
251
  }
252
+ if (request.have_request_timing) {
253
+ request.request_end = Timestamp::GetCurrentTimestamp();
254
+ }
237
255
  LogRequest(request, response ? response.get() : nullptr);
238
256
  return response;
239
257
  });
@@ -2,6 +2,7 @@
2
2
 
3
3
  #include "duckdb/common/enum_util.hpp"
4
4
  #include "duckdb/main/query_profiler.hpp"
5
+ #include "duckdb/logging/log_manager.hpp"
5
6
 
6
7
  #include "yyjson.hpp"
7
8
 
@@ -169,6 +170,16 @@ string ProfilingInfo::GetMetricAsString(const MetricsType metric) const {
169
170
  return metrics.at(metric).ToString();
170
171
  }
171
172
 
173
+ void ProfilingInfo::WriteMetricsToLog(ClientContext &context) {
174
+ auto &logger = Logger::Get(context);
175
+ if (logger.ShouldLog(MetricsLogType::NAME, MetricsLogType::LEVEL)) {
176
+ for (auto &metric : settings) {
177
+ logger.WriteLog(MetricsLogType::NAME, MetricsLogType::LEVEL,
178
+ MetricsLogType::ConstructLogMessage(metric, metrics[metric]));
179
+ }
180
+ }
181
+ }
182
+
172
183
  void ProfilingInfo::WriteMetricsToJSON(yyjson_mut_doc *doc, yyjson_mut_val *dest) {
173
184
  for (auto &metric : settings) {
174
185
  auto metric_str = StringUtil::Lower(EnumUtil::ToString(metric));
@@ -297,6 +297,9 @@ void QueryProfiler::EndQuery() {
297
297
 
298
298
  guard.unlock();
299
299
 
300
+ // To log is inexpensive, whether to log or not depends on whether logging is active
301
+ ToLog();
302
+
300
303
  if (emit_output) {
301
304
  string tree = ToString();
302
305
  auto save_location = GetSaveLocation();
@@ -797,6 +800,19 @@ static string StringifyAndFree(yyjson_mut_doc *doc, yyjson_mut_val *object) {
797
800
  return result;
798
801
  }
799
802
 
803
+ void QueryProfiler::ToLog() const {
804
+ lock_guard<std::mutex> guard(lock);
805
+
806
+ if (!root) {
807
+ // No root, not much to do
808
+ return;
809
+ }
810
+
811
+ auto &settings = root->GetProfilingInfo();
812
+
813
+ settings.WriteMetricsToLog(context);
814
+ }
815
+
800
816
  string QueryProfiler::ToJSON() const {
801
817
  lock_guard<std::mutex> guard(lock);
802
818
  auto doc = yyjson_mut_doc_new(nullptr);
@@ -14,12 +14,21 @@ CreateTableRelation::CreateTableRelation(shared_ptr<Relation> child_p, string sc
14
14
  TryBindRelation(columns);
15
15
  }
16
16
 
17
+ CreateTableRelation::CreateTableRelation(shared_ptr<Relation> child_p, string catalog_name, string schema_name,
18
+ string table_name, bool temporary_p, OnCreateConflict on_conflict)
19
+ : Relation(child_p->context, RelationType::CREATE_TABLE_RELATION), child(std::move(child_p)),
20
+ catalog_name(std::move(catalog_name)), schema_name(std::move(schema_name)), table_name(std::move(table_name)),
21
+ temporary(temporary_p), on_conflict(on_conflict) {
22
+ TryBindRelation(columns);
23
+ }
24
+
17
25
  BoundStatement CreateTableRelation::Bind(Binder &binder) {
18
26
  auto select = make_uniq<SelectStatement>();
19
27
  select->node = child->GetQueryNode();
20
28
 
21
29
  CreateStatement stmt;
22
30
  auto info = make_uniq<CreateTableInfo>();
31
+ info->catalog = catalog_name;
23
32
  info->schema = schema_name;
24
33
  info->table = table_name;
25
34
  info->query = std::move(select);
@@ -13,11 +13,18 @@ InsertRelation::InsertRelation(shared_ptr<Relation> child_p, string schema_name,
13
13
  TryBindRelation(columns);
14
14
  }
15
15
 
16
+ InsertRelation::InsertRelation(shared_ptr<Relation> child_p, string catalog_name, string schema_name, string table_name)
17
+ : Relation(child_p->context, RelationType::INSERT_RELATION), child(std::move(child_p)),
18
+ catalog_name(std::move(catalog_name)), schema_name(std::move(schema_name)), table_name(std::move(table_name)) {
19
+ TryBindRelation(columns);
20
+ }
21
+
16
22
  BoundStatement InsertRelation::Bind(Binder &binder) {
17
23
  InsertStatement stmt;
18
24
  auto select = make_uniq<SelectStatement>();
19
25
  select->node = child->GetQueryNode();
20
26
 
27
+ stmt.catalog = catalog_name;
21
28
  stmt.schema = schema_name;
22
29
  stmt.table = table_name;
23
30
  stmt.select_statement = std::move(select);
@@ -3,6 +3,7 @@
3
3
  #include "duckdb/parser/query_node/select_node.hpp"
4
4
  #include "duckdb/parser/expression/star_expression.hpp"
5
5
  #include "duckdb/main/relation/delete_relation.hpp"
6
+ #include "duckdb/main/relation/value_relation.hpp"
6
7
  #include "duckdb/main/relation/update_relation.hpp"
7
8
  #include "duckdb/parser/parser.hpp"
8
9
  #include "duckdb/main/client_context.hpp"
@@ -87,4 +88,17 @@ void TableRelation::Delete(const string &condition) {
87
88
  del->Execute();
88
89
  }
89
90
 
91
+ void TableRelation::Insert(const vector<vector<Value>> &values) {
92
+ vector<string> column_names;
93
+ auto rel = make_shared_ptr<ValueRelation>(context->GetContext(), values, std::move(column_names), "values");
94
+ rel->Insert(description->database, description->schema, description->table);
95
+ }
96
+
97
+ void TableRelation::Insert(vector<vector<unique_ptr<ParsedExpression>>> &&expressions) {
98
+ vector<string> column_names;
99
+ auto rel = make_shared_ptr<ValueRelation>(context->GetContext(), std::move(expressions), std::move(column_names),
100
+ "values");
101
+ rel->Insert(description->database, description->schema, description->table);
102
+ }
103
+
90
104
  } // namespace duckdb
@@ -241,7 +241,12 @@ BoundStatement Relation::Bind(Binder &binder) {
241
241
  }
242
242
 
243
243
  shared_ptr<Relation> Relation::InsertRel(const string &schema_name, const string &table_name) {
244
- return make_shared_ptr<InsertRelation>(shared_from_this(), schema_name, table_name);
244
+ return InsertRel(INVALID_CATALOG, schema_name, table_name);
245
+ }
246
+
247
+ shared_ptr<Relation> Relation::InsertRel(const string &catalog_name, const string &schema_name,
248
+ const string &table_name) {
249
+ return make_shared_ptr<InsertRelation>(shared_from_this(), catalog_name, schema_name, table_name);
245
250
  }
246
251
 
247
252
  void Relation::Insert(const string &table_name) {
@@ -249,7 +254,11 @@ void Relation::Insert(const string &table_name) {
249
254
  }
250
255
 
251
256
  void Relation::Insert(const string &schema_name, const string &table_name) {
252
- auto insert = InsertRel(schema_name, table_name);
257
+ Insert(INVALID_CATALOG, schema_name, table_name);
258
+ }
259
+
260
+ void Relation::Insert(const string &catalog_name, const string &schema_name, const string &table_name) {
261
+ auto insert = InsertRel(catalog_name, schema_name, table_name);
253
262
  auto res = insert->Execute();
254
263
  if (res->HasError()) {
255
264
  const string prepended_message = "Failed to insert into table '" + table_name + "': ";
@@ -258,30 +267,37 @@ void Relation::Insert(const string &schema_name, const string &table_name) {
258
267
  }
259
268
 
260
269
  void Relation::Insert(const vector<vector<Value>> &values) {
261
- vector<string> column_names;
262
- auto rel = make_shared_ptr<ValueRelation>(context->GetContext(), values, std::move(column_names), "values");
263
- rel->Insert(GetAlias());
270
+ throw InvalidInputException("INSERT with values can only be used on base tables!");
264
271
  }
265
272
 
266
273
  void Relation::Insert(vector<vector<unique_ptr<ParsedExpression>>> &&expressions) {
267
- vector<string> column_names;
268
- auto rel = make_shared_ptr<ValueRelation>(context->GetContext(), std::move(expressions), std::move(column_names),
269
- "values");
270
- rel->Insert(GetAlias());
274
+ (void)std::move(expressions);
275
+ throw InvalidInputException("INSERT with expressions can only be used on base tables!");
271
276
  }
272
277
 
273
278
  shared_ptr<Relation> Relation::CreateRel(const string &schema_name, const string &table_name, bool temporary,
274
279
  OnCreateConflict on_conflict) {
275
- return make_shared_ptr<CreateTableRelation>(shared_from_this(), schema_name, table_name, temporary, on_conflict);
280
+ return CreateRel(INVALID_CATALOG, schema_name, table_name, temporary, on_conflict);
281
+ }
282
+
283
+ shared_ptr<Relation> Relation::CreateRel(const string &catalog_name, const string &schema_name,
284
+ const string &table_name, bool temporary, OnCreateConflict on_conflict) {
285
+ return make_shared_ptr<CreateTableRelation>(shared_from_this(), catalog_name, schema_name, table_name, temporary,
286
+ on_conflict);
276
287
  }
277
288
 
278
289
  void Relation::Create(const string &table_name, bool temporary, OnCreateConflict on_conflict) {
279
- Create(INVALID_SCHEMA, table_name, temporary, on_conflict);
290
+ Create(INVALID_CATALOG, INVALID_SCHEMA, table_name, temporary, on_conflict);
280
291
  }
281
292
 
282
293
  void Relation::Create(const string &schema_name, const string &table_name, bool temporary,
283
294
  OnCreateConflict on_conflict) {
284
- auto create = CreateRel(schema_name, table_name, temporary, on_conflict);
295
+ Create(INVALID_CATALOG, schema_name, table_name, temporary, on_conflict);
296
+ }
297
+
298
+ void Relation::Create(const string &catalog_name, const string &schema_name, const string &table_name, bool temporary,
299
+ OnCreateConflict on_conflict) {
300
+ auto create = CreateRel(catalog_name, schema_name, table_name, temporary, on_conflict);
285
301
  auto res = create->Execute();
286
302
  if (res->HasError()) {
287
303
  const string prepended_message = "Failed to create table '" + table_name + "': ";
@@ -961,9 +961,15 @@ void ForceCompressionSetting::SetGlobal(DatabaseInstance *db, DBConfig &config,
961
961
  } else {
962
962
  auto compression_type = CompressionTypeFromString(compression);
963
963
  //! FIXME: do we want to try to retrieve the AttachedDatabase here to get the StorageManager ??
964
- if (CompressionTypeIsDeprecated(compression_type)) {
965
- throw ParserException("Attempted to force a deprecated compression type (%s)",
966
- CompressionTypeToString(compression_type));
964
+ auto compression_availability_result = CompressionTypeIsAvailable(compression_type);
965
+ if (!compression_availability_result.IsAvailable()) {
966
+ if (compression_availability_result.IsDeprecated()) {
967
+ throw ParserException("Attempted to force a deprecated compression type (%s)",
968
+ CompressionTypeToString(compression_type));
969
+ } else {
970
+ throw ParserException("Attempted to force a compression type that isn't available yet (%s)",
971
+ CompressionTypeToString(compression_type));
972
+ }
967
973
  }
968
974
  if (compression_type == CompressionType::COMPRESSION_AUTO) {
969
975
  auto compression_types = StringUtil::Join(ListCompressionTypes(), ", ");
@@ -6,6 +6,7 @@
6
6
  #include "duckdb/planner/operator/logical_comparison_join.hpp"
7
7
  #include "duckdb/planner/operator/logical_cross_product.hpp"
8
8
  #include "duckdb/planner/operator/logical_join.hpp"
9
+ #include "duckdb/planner/operator/logical_distinct.hpp"
9
10
 
10
11
  namespace duckdb {
11
12
 
@@ -26,6 +27,7 @@ unique_ptr<LogicalOperator> FilterPullup::Rewrite(unique_ptr<LogicalOperator> op
26
27
  case LogicalOperatorType::LOGICAL_EXCEPT:
27
28
  return PullupSetOperation(std::move(op));
28
29
  case LogicalOperatorType::LOGICAL_DISTINCT:
30
+ return PullupDistinct(std::move(op));
29
31
  case LogicalOperatorType::LOGICAL_ORDER_BY: {
30
32
  // we can just pull directly through these operations without any rewriting
31
33
  op->children[0] = Rewrite(std::move(op->children[0]));
@@ -115,6 +117,18 @@ unique_ptr<LogicalOperator> FilterPullup::PullupCrossProduct(unique_ptr<LogicalO
115
117
  return PullupBothSide(std::move(op));
116
118
  }
117
119
 
120
+ unique_ptr<LogicalOperator> FilterPullup::PullupDistinct(unique_ptr<LogicalOperator> op) {
121
+ const auto &distinct = op->Cast<LogicalDistinct>();
122
+ if (distinct.distinct_type == DistinctType::DISTINCT) {
123
+ // Can pull up through a DISTINCT
124
+ op->children[0] = Rewrite(std::move(op->children[0]));
125
+ return op;
126
+ }
127
+ // Cannot pull up through a DISTINCT ON (see #19327)
128
+ D_ASSERT(distinct.distinct_type == DistinctType::DISTINCT_ON);
129
+ return FinishPullup(std::move(op));
130
+ }
131
+
118
132
  unique_ptr<LogicalOperator> FilterPullup::GeneratePullupFilter(unique_ptr<LogicalOperator> child,
119
133
  vector<unique_ptr<Expression>> &expressions) {
120
134
  unique_ptr<LogicalFilter> filter = make_uniq<LogicalFilter>();