duckdb 1.4.0 → 1.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (144) hide show
  1. package/package.json +1 -1
  2. package/src/duckdb/extension/core_functions/scalar/generic/current_setting.cpp +1 -4
  3. package/src/duckdb/extension/icu/icu-strptime.cpp +2 -1
  4. package/src/duckdb/extension/json/include/json_common.hpp +2 -4
  5. package/src/duckdb/extension/json/json_functions.cpp +5 -1
  6. package/src/duckdb/extension/parquet/column_writer.cpp +31 -21
  7. package/src/duckdb/extension/parquet/geo_parquet.cpp +21 -6
  8. package/src/duckdb/extension/parquet/include/column_writer.hpp +2 -2
  9. package/src/duckdb/extension/parquet/include/geo_parquet.hpp +28 -1
  10. package/src/duckdb/extension/parquet/include/parquet_writer.hpp +7 -2
  11. package/src/duckdb/extension/parquet/include/reader/string_column_reader.hpp +13 -0
  12. package/src/duckdb/extension/parquet/include/writer/array_column_writer.hpp +4 -0
  13. package/src/duckdb/extension/parquet/parquet_extension.cpp +56 -1
  14. package/src/duckdb/extension/parquet/parquet_reader.cpp +4 -1
  15. package/src/duckdb/extension/parquet/parquet_statistics.cpp +5 -7
  16. package/src/duckdb/extension/parquet/parquet_writer.cpp +15 -8
  17. package/src/duckdb/extension/parquet/reader/string_column_reader.cpp +17 -4
  18. package/src/duckdb/extension/parquet/writer/array_column_writer.cpp +22 -28
  19. package/src/duckdb/extension/parquet/writer/primitive_column_writer.cpp +17 -5
  20. package/src/duckdb/extension/parquet/writer/struct_column_writer.cpp +3 -2
  21. package/src/duckdb/src/catalog/catalog_search_path.cpp +2 -2
  22. package/src/duckdb/src/catalog/catalog_set.cpp +1 -2
  23. package/src/duckdb/src/common/enum_util.cpp +20 -0
  24. package/src/duckdb/src/common/file_system.cpp +0 -30
  25. package/src/duckdb/src/common/sorting/sort.cpp +25 -6
  26. package/src/duckdb/src/common/sorting/sorted_run_merger.cpp +1 -0
  27. package/src/duckdb/src/common/string_util.cpp +24 -0
  28. package/src/duckdb/src/common/virtual_file_system.cpp +59 -10
  29. package/src/duckdb/src/execution/index/art/art_merger.cpp +0 -3
  30. package/src/duckdb/src/execution/index/art/prefix.cpp +4 -0
  31. package/src/duckdb/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +1 -1
  32. package/src/duckdb/src/execution/operator/helper/physical_reset.cpp +2 -2
  33. package/src/duckdb/src/execution/operator/schema/physical_attach.cpp +1 -1
  34. package/src/duckdb/src/execution/physical_plan/plan_asof_join.cpp +3 -3
  35. package/src/duckdb/src/function/table/system/duckdb_connection_count.cpp +45 -0
  36. package/src/duckdb/src/function/table/system/duckdb_settings.cpp +11 -1
  37. package/src/duckdb/src/function/table/system_functions.cpp +1 -0
  38. package/src/duckdb/src/function/table/version/pragma_version.cpp +3 -3
  39. package/src/duckdb/src/include/duckdb/common/enum_util.hpp +8 -0
  40. package/src/duckdb/src/include/duckdb/common/string_util.hpp +2 -0
  41. package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp +4 -1
  42. package/src/duckdb/src/include/duckdb/function/scalar/variant_functions.hpp +1 -1
  43. package/src/duckdb/src/include/duckdb/function/table/system_functions.hpp +4 -0
  44. package/src/duckdb/src/include/duckdb/logging/log_storage.hpp +6 -6
  45. package/src/duckdb/src/include/duckdb/logging/log_type.hpp +26 -3
  46. package/src/duckdb/src/include/duckdb/main/attached_database.hpp +4 -0
  47. package/src/duckdb/src/include/duckdb/main/client_context.hpp +2 -0
  48. package/src/duckdb/src/include/duckdb/main/connection.hpp +0 -1
  49. package/src/duckdb/src/include/duckdb/main/connection_manager.hpp +0 -1
  50. package/src/duckdb/src/include/duckdb/main/database_file_path_manager.hpp +12 -1
  51. package/src/duckdb/src/include/duckdb/main/database_manager.hpp +3 -0
  52. package/src/duckdb/src/include/duckdb/main/relation/create_table_relation.hpp +2 -0
  53. package/src/duckdb/src/include/duckdb/main/relation/create_view_relation.hpp +2 -0
  54. package/src/duckdb/src/include/duckdb/main/relation/delete_relation.hpp +2 -0
  55. package/src/duckdb/src/include/duckdb/main/relation/explain_relation.hpp +2 -0
  56. package/src/duckdb/src/include/duckdb/main/relation/insert_relation.hpp +2 -0
  57. package/src/duckdb/src/include/duckdb/main/relation/query_relation.hpp +1 -0
  58. package/src/duckdb/src/include/duckdb/main/relation/update_relation.hpp +2 -0
  59. package/src/duckdb/src/include/duckdb/main/relation/write_csv_relation.hpp +2 -0
  60. package/src/duckdb/src/include/duckdb/main/relation/write_parquet_relation.hpp +2 -0
  61. package/src/duckdb/src/include/duckdb/main/relation.hpp +2 -1
  62. package/src/duckdb/src/include/duckdb/main/secret/secret.hpp +3 -1
  63. package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp +3 -2
  64. package/src/duckdb/src/include/duckdb/planner/binder.hpp +62 -3
  65. package/src/duckdb/src/include/duckdb/planner/expression_binder/lateral_binder.hpp +2 -2
  66. package/src/duckdb/src/include/duckdb/planner/operator/logical_cte.hpp +1 -1
  67. package/src/duckdb/src/include/duckdb/planner/operator/logical_dependent_join.hpp +3 -3
  68. package/src/duckdb/src/include/duckdb/planner/subquery/flatten_dependent_join.hpp +2 -2
  69. package/src/duckdb/src/include/duckdb/planner/subquery/has_correlated_expressions.hpp +2 -2
  70. package/src/duckdb/src/include/duckdb/planner/subquery/rewrite_cte_scan.hpp +2 -2
  71. package/src/duckdb/src/include/duckdb/planner/tableref/bound_joinref.hpp +1 -1
  72. package/src/duckdb/src/include/duckdb/storage/compression/alp/alp_analyze.hpp +6 -1
  73. package/src/duckdb/src/include/duckdb/storage/compression/alprd/alprd_analyze.hpp +5 -1
  74. package/src/duckdb/src/include/duckdb/storage/metadata/metadata_manager.hpp +9 -7
  75. package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp +2 -0
  76. package/src/duckdb/src/include/duckdb/storage/table/array_column_data.hpp +4 -4
  77. package/src/duckdb/src/include/duckdb/storage/table/column_data.hpp +6 -6
  78. package/src/duckdb/src/include/duckdb/storage/table/list_column_data.hpp +4 -4
  79. package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +4 -4
  80. package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +5 -3
  81. package/src/duckdb/src/include/duckdb/storage/table/row_id_column_data.hpp +4 -4
  82. package/src/duckdb/src/include/duckdb/storage/table/standard_column_data.hpp +4 -4
  83. package/src/duckdb/src/include/duckdb/storage/table/struct_column_data.hpp +4 -4
  84. package/src/duckdb/src/include/duckdb/storage/table/update_segment.hpp +2 -2
  85. package/src/duckdb/src/include/duckdb/transaction/duck_transaction.hpp +2 -1
  86. package/src/duckdb/src/include/duckdb/transaction/update_info.hpp +4 -1
  87. package/src/duckdb/src/include/duckdb/transaction/wal_write_state.hpp +1 -1
  88. package/src/duckdb/src/logging/log_types.cpp +63 -0
  89. package/src/duckdb/src/main/attached_database.cpp +16 -3
  90. package/src/duckdb/src/main/client_context.cpp +27 -19
  91. package/src/duckdb/src/main/connection.cpp +2 -5
  92. package/src/duckdb/src/main/database_file_path_manager.cpp +23 -6
  93. package/src/duckdb/src/main/database_manager.cpp +18 -3
  94. package/src/duckdb/src/main/http/http_util.cpp +3 -1
  95. package/src/duckdb/src/main/relation/create_table_relation.cpp +8 -0
  96. package/src/duckdb/src/main/relation/create_view_relation.cpp +8 -0
  97. package/src/duckdb/src/main/relation/delete_relation.cpp +8 -0
  98. package/src/duckdb/src/main/relation/explain_relation.cpp +8 -0
  99. package/src/duckdb/src/main/relation/insert_relation.cpp +8 -0
  100. package/src/duckdb/src/main/relation/query_relation.cpp +4 -0
  101. package/src/duckdb/src/main/relation/update_relation.cpp +8 -0
  102. package/src/duckdb/src/main/relation/write_csv_relation.cpp +8 -0
  103. package/src/duckdb/src/main/relation/write_parquet_relation.cpp +8 -0
  104. package/src/duckdb/src/main/relation.cpp +2 -2
  105. package/src/duckdb/src/optimizer/filter_combiner.cpp +7 -0
  106. package/src/duckdb/src/optimizer/filter_pushdown.cpp +9 -3
  107. package/src/duckdb/src/optimizer/pushdown/pushdown_get.cpp +4 -1
  108. package/src/duckdb/src/optimizer/rule/comparison_simplification.cpp +3 -7
  109. package/src/duckdb/src/parser/statement/relation_statement.cpp +1 -4
  110. package/src/duckdb/src/parser/transform/statement/transform_create_function.cpp +2 -0
  111. package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp +8 -6
  112. package/src/duckdb/src/planner/binder/statement/bind_create.cpp +1 -5
  113. package/src/duckdb/src/planner/binder/statement/bind_merge_into.cpp +10 -2
  114. package/src/duckdb/src/planner/binder/statement/bind_pragma.cpp +20 -3
  115. package/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp +8 -3
  116. package/src/duckdb/src/planner/binder/tableref/bind_table_function.cpp +9 -2
  117. package/src/duckdb/src/planner/binder.cpp +2 -2
  118. package/src/duckdb/src/planner/expression_binder/lateral_binder.cpp +9 -13
  119. package/src/duckdb/src/planner/expression_binder/table_function_binder.cpp +4 -0
  120. package/src/duckdb/src/planner/expression_binder.cpp +3 -1
  121. package/src/duckdb/src/planner/operator/logical_dependent_join.cpp +2 -2
  122. package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp +12 -14
  123. package/src/duckdb/src/planner/subquery/has_correlated_expressions.cpp +1 -1
  124. package/src/duckdb/src/planner/subquery/rewrite_cte_scan.cpp +2 -2
  125. package/src/duckdb/src/storage/compression/bitpacking.cpp +1 -2
  126. package/src/duckdb/src/storage/data_table.cpp +2 -2
  127. package/src/duckdb/src/storage/local_storage.cpp +1 -1
  128. package/src/duckdb/src/storage/metadata/metadata_manager.cpp +67 -25
  129. package/src/duckdb/src/storage/statistics/string_stats.cpp +8 -0
  130. package/src/duckdb/src/storage/table/array_column_data.cpp +6 -5
  131. package/src/duckdb/src/storage/table/column_data.cpp +23 -9
  132. package/src/duckdb/src/storage/table/column_data_checkpointer.cpp +15 -1
  133. package/src/duckdb/src/storage/table/list_column_data.cpp +5 -4
  134. package/src/duckdb/src/storage/table/row_group.cpp +8 -8
  135. package/src/duckdb/src/storage/table/row_group_collection.cpp +12 -8
  136. package/src/duckdb/src/storage/table/row_id_column_data.cpp +5 -4
  137. package/src/duckdb/src/storage/table/standard_column_data.cpp +9 -8
  138. package/src/duckdb/src/storage/table/struct_column_data.cpp +10 -9
  139. package/src/duckdb/src/storage/table/update_segment.cpp +12 -10
  140. package/src/duckdb/src/transaction/commit_state.cpp +18 -0
  141. package/src/duckdb/src/transaction/duck_transaction.cpp +2 -10
  142. package/src/duckdb/src/transaction/wal_write_state.cpp +5 -5
  143. package/src/duckdb/third_party/httplib/httplib.hpp +6 -1
  144. package/src/duckdb/ub_src_function_table_system.cpp +2 -0
@@ -71,6 +71,8 @@ struct StringStats {
71
71
  ExpressionType comparison_type, const string &value);
72
72
 
73
73
  DUCKDB_API static void Update(BaseStatistics &stats, const string_t &value);
74
+ DUCKDB_API static void SetMin(BaseStatistics &stats, const string_t &value);
75
+ DUCKDB_API static void SetMax(BaseStatistics &stats, const string_t &value);
74
76
  DUCKDB_API static void Merge(BaseStatistics &stats, const BaseStatistics &other);
75
77
  DUCKDB_API static void Verify(const BaseStatistics &stats, Vector &vector, const SelectionVector &sel, idx_t count);
76
78
 
@@ -48,10 +48,10 @@ public:
48
48
  idx_t Fetch(ColumnScanState &state, row_t row_id, Vector &result) override;
49
49
  void FetchRow(TransactionData transaction, ColumnFetchState &state, row_t row_id, Vector &result,
50
50
  idx_t result_idx) override;
51
- void Update(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids,
52
- idx_t update_count) override;
53
- void UpdateColumn(TransactionData transaction, const vector<column_t> &column_path, Vector &update_vector,
54
- row_t *row_ids, idx_t update_count, idx_t depth) override;
51
+ void Update(TransactionData transaction, DataTable &data_table, idx_t column_index, Vector &update_vector,
52
+ row_t *row_ids, idx_t update_count) override;
53
+ void UpdateColumn(TransactionData transaction, DataTable &data_table, const vector<column_t> &column_path,
54
+ Vector &update_vector, row_t *row_ids, idx_t update_count, idx_t depth) override;
55
55
  unique_ptr<BaseStatistics> GetUpdateStatistics() override;
56
56
 
57
57
  void CommitDropColumn() override;
@@ -154,10 +154,10 @@ public:
154
154
  virtual void FetchRow(TransactionData transaction, ColumnFetchState &state, row_t row_id, Vector &result,
155
155
  idx_t result_idx);
156
156
 
157
- virtual void Update(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids,
158
- idx_t update_count);
159
- virtual void UpdateColumn(TransactionData transaction, const vector<column_t> &column_path, Vector &update_vector,
160
- row_t *row_ids, idx_t update_count, idx_t depth);
157
+ virtual void Update(TransactionData transaction, DataTable &data_table, idx_t column_index, Vector &update_vector,
158
+ row_t *row_ids, idx_t update_count);
159
+ virtual void UpdateColumn(TransactionData transaction, DataTable &data_table, const vector<column_t> &column_path,
160
+ Vector &update_vector, row_t *row_ids, idx_t update_count, idx_t depth);
161
161
  virtual unique_ptr<BaseStatistics> GetUpdateStatistics();
162
162
 
163
163
  virtual void CommitDropColumn();
@@ -217,8 +217,8 @@ protected:
217
217
  void FetchUpdates(TransactionData transaction, idx_t vector_index, Vector &result, idx_t scan_count,
218
218
  bool allow_updates, bool scan_committed);
219
219
  void FetchUpdateRow(TransactionData transaction, row_t row_id, Vector &result, idx_t result_idx);
220
- void UpdateInternal(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids,
221
- idx_t update_count, Vector &base_vector);
220
+ void UpdateInternal(TransactionData transaction, DataTable &data_table, idx_t column_index, Vector &update_vector,
221
+ row_t *row_ids, idx_t update_count, Vector &base_vector);
222
222
  idx_t FetchUpdateData(ColumnScanState &state, row_t *row_ids, Vector &base_vector);
223
223
 
224
224
  idx_t GetVectorCount(idx_t vector_index) const;
@@ -46,10 +46,10 @@ public:
46
46
  idx_t Fetch(ColumnScanState &state, row_t row_id, Vector &result) override;
47
47
  void FetchRow(TransactionData transaction, ColumnFetchState &state, row_t row_id, Vector &result,
48
48
  idx_t result_idx) override;
49
- void Update(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids,
50
- idx_t update_count) override;
51
- void UpdateColumn(TransactionData transaction, const vector<column_t> &column_path, Vector &update_vector,
52
- row_t *row_ids, idx_t update_count, idx_t depth) override;
49
+ void Update(TransactionData transaction, DataTable &data_table, idx_t column_index, Vector &update_vector,
50
+ row_t *row_ids, idx_t update_count) override;
51
+ void UpdateColumn(TransactionData transaction, DataTable &data_table, const vector<column_t> &column_path,
52
+ Vector &update_vector, row_t *row_ids, idx_t update_count, idx_t depth) override;
53
53
  unique_ptr<BaseStatistics> GetUpdateStatistics() override;
54
54
 
55
55
  void CommitDropColumn() override;
@@ -160,12 +160,12 @@ public:
160
160
  void InitializeAppend(RowGroupAppendState &append_state);
161
161
  void Append(RowGroupAppendState &append_state, DataChunk &chunk, idx_t append_count);
162
162
 
163
- void Update(TransactionData transaction, DataChunk &updates, row_t *ids, idx_t offset, idx_t count,
164
- const vector<PhysicalIndex> &column_ids);
163
+ void Update(TransactionData transaction, DataTable &data_table, DataChunk &updates, row_t *ids, idx_t offset,
164
+ idx_t count, const vector<PhysicalIndex> &column_ids);
165
165
  //! Update a single column; corresponds to DataTable::UpdateColumn
166
166
  //! This method should only be called from the WAL
167
- void UpdateColumn(TransactionData transaction, DataChunk &updates, Vector &row_ids, idx_t offset, idx_t count,
168
- const vector<column_t> &column_path);
167
+ void UpdateColumn(TransactionData transaction, DataTable &data_table, DataChunk &updates, Vector &row_ids,
168
+ idx_t offset, idx_t count, const vector<column_t> &column_path);
169
169
 
170
170
  void MergeStatistics(idx_t column_idx, const BaseStatistics &other);
171
171
  void MergeIntoStatistics(idx_t column_idx, BaseStatistics &other);
@@ -36,6 +36,7 @@ struct CollectionCheckpointState;
36
36
  struct PersistentCollectionData;
37
37
  class CheckpointTask;
38
38
  class TableIOManager;
39
+ class DataTable;
39
40
 
40
41
  class RowGroupCollection {
41
42
  public:
@@ -99,9 +100,10 @@ public:
99
100
  void RemoveFromIndexes(TableIndexList &indexes, Vector &row_identifiers, idx_t count);
100
101
 
101
102
  idx_t Delete(TransactionData transaction, DataTable &table, row_t *ids, idx_t count);
102
- void Update(TransactionData transaction, row_t *ids, const vector<PhysicalIndex> &column_ids, DataChunk &updates);
103
- void UpdateColumn(TransactionData transaction, Vector &row_ids, const vector<column_t> &column_path,
104
- DataChunk &updates);
103
+ void Update(TransactionData transaction, DataTable &table, row_t *ids, const vector<PhysicalIndex> &column_ids,
104
+ DataChunk &updates);
105
+ void UpdateColumn(TransactionData transaction, DataTable &table, Vector &row_ids,
106
+ const vector<column_t> &column_path, DataChunk &updates);
105
107
 
106
108
  void Checkpoint(TableDataWriter &writer, TableStatistics &global_stats);
107
109
 
@@ -48,10 +48,10 @@ public:
48
48
  void AppendData(BaseStatistics &stats, ColumnAppendState &state, UnifiedVectorFormat &vdata, idx_t count) override;
49
49
  void RevertAppend(row_t start_row) override;
50
50
 
51
- void Update(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids,
52
- idx_t update_count) override;
53
- void UpdateColumn(TransactionData transaction, const vector<column_t> &column_path, Vector &update_vector,
54
- row_t *row_ids, idx_t update_count, idx_t depth) override;
51
+ void Update(TransactionData transaction, DataTable &data_table, idx_t column_index, Vector &update_vector,
52
+ row_t *row_ids, idx_t update_count) override;
53
+ void UpdateColumn(TransactionData transaction, DataTable &data_table, const vector<column_t> &column_path,
54
+ Vector &update_vector, row_t *row_ids, idx_t update_count, idx_t depth) override;
55
55
 
56
56
  void CommitDropColumn() override;
57
57
 
@@ -47,10 +47,10 @@ public:
47
47
  idx_t Fetch(ColumnScanState &state, row_t row_id, Vector &result) override;
48
48
  void FetchRow(TransactionData transaction, ColumnFetchState &state, row_t row_id, Vector &result,
49
49
  idx_t result_idx) override;
50
- void Update(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids,
51
- idx_t update_count) override;
52
- void UpdateColumn(TransactionData transaction, const vector<column_t> &column_path, Vector &update_vector,
53
- row_t *row_ids, idx_t update_count, idx_t depth) override;
50
+ void Update(TransactionData transaction, DataTable &data_table, idx_t column_index, Vector &update_vector,
51
+ row_t *row_ids, idx_t update_count) override;
52
+ void UpdateColumn(TransactionData transaction, DataTable &data_table, const vector<column_t> &column_path,
53
+ Vector &update_vector, row_t *row_ids, idx_t update_count, idx_t depth) override;
54
54
  unique_ptr<BaseStatistics> GetUpdateStatistics() override;
55
55
 
56
56
  void CommitDropColumn() override;
@@ -46,10 +46,10 @@ public:
46
46
  idx_t Fetch(ColumnScanState &state, row_t row_id, Vector &result) override;
47
47
  void FetchRow(TransactionData transaction, ColumnFetchState &state, row_t row_id, Vector &result,
48
48
  idx_t result_idx) override;
49
- void Update(TransactionData transaction, idx_t column_index, Vector &update_vector, row_t *row_ids,
50
- idx_t update_count) override;
51
- void UpdateColumn(TransactionData transaction, const vector<column_t> &column_path, Vector &update_vector,
52
- row_t *row_ids, idx_t update_count, idx_t depth) override;
49
+ void Update(TransactionData transaction, DataTable &data_table, idx_t column_index, Vector &update_vector,
50
+ row_t *row_ids, idx_t update_count) override;
51
+ void UpdateColumn(TransactionData transaction, DataTable &data_table, const vector<column_t> &column_path,
52
+ Vector &update_vector, row_t *row_ids, idx_t update_count, idx_t depth) override;
53
53
  unique_ptr<BaseStatistics> GetUpdateStatistics() override;
54
54
 
55
55
  void CommitDropColumn() override;
@@ -38,8 +38,8 @@ public:
38
38
  void FetchUpdates(TransactionData transaction, idx_t vector_index, Vector &result);
39
39
  void FetchCommitted(idx_t vector_index, Vector &result);
40
40
  void FetchCommittedRange(idx_t start_row, idx_t count, Vector &result);
41
- void Update(TransactionData transaction, idx_t column_index, Vector &update, row_t *ids, idx_t count,
42
- Vector &base_data);
41
+ void Update(TransactionData transaction, DataTable &data_table, idx_t column_index, Vector &update, row_t *ids,
42
+ idx_t count, Vector &base_data);
43
43
  void FetchRow(TransactionData transaction, idx_t row_id, Vector &result, idx_t result_idx);
44
44
 
45
45
  void RollbackUpdate(UpdateInfo &info);
@@ -76,7 +76,7 @@ public:
76
76
  idx_t base_row);
77
77
  void PushSequenceUsage(SequenceCatalogEntry &entry, const SequenceData &data);
78
78
  void PushAppend(DataTable &table, idx_t row_start, idx_t row_count);
79
- UndoBufferReference CreateUpdateInfo(idx_t type_size, idx_t entries);
79
+ UndoBufferReference CreateUpdateInfo(idx_t type_size, DataTable &data_table, idx_t entries);
80
80
 
81
81
  bool IsDuckTransaction() const override {
82
82
  return true;
@@ -90,6 +90,7 @@ public:
90
90
  //! Get a shared lock on a table
91
91
  shared_ptr<CheckpointLock> SharedLockTable(DataTableInfo &info);
92
92
 
93
+ //! Hold an owning reference of the table, needed to safely reference it inside the transaction commit/undo logic
93
94
  void ModifyTable(DataTable &tbl);
94
95
 
95
96
  private:
@@ -17,6 +17,7 @@
17
17
  namespace duckdb {
18
18
  class UpdateSegment;
19
19
  struct DataTableInfo;
20
+ class DataTable;
20
21
 
21
22
  //! UpdateInfo is a class that represents a set of updates applied to a single vector.
22
23
  //! The UpdateInfo struct contains metadata associated with the update.
@@ -26,6 +27,8 @@ struct DataTableInfo;
26
27
  struct UpdateInfo {
27
28
  //! The update segment that this update info affects
28
29
  UpdateSegment *segment;
30
+ //! The table this was update was made on
31
+ DataTable *table;
29
32
  //! The column index of which column we are updating
30
33
  idx_t column_index;
31
34
  //! The version number
@@ -87,7 +90,7 @@ struct UpdateInfo {
87
90
  //! Returns the total allocation size for an UpdateInfo entry, together with space for the tuple data
88
91
  static idx_t GetAllocSize(idx_t type_size);
89
92
  //! Initialize an UpdateInfo struct that has been allocated using GetAllocSize (i.e. has extra space after it)
90
- static void Initialize(UpdateInfo &info, transaction_t transaction_id);
93
+ static void Initialize(UpdateInfo &info, DataTable &data_table, transaction_t transaction_id);
91
94
  };
92
95
 
93
96
  } // namespace duckdb
@@ -31,7 +31,7 @@ public:
31
31
  void CommitEntry(UndoFlags type, data_ptr_t data);
32
32
 
33
33
  private:
34
- void SwitchTable(DataTableInfo *table, UndoFlags new_op);
34
+ void SwitchTable(DataTableInfo &table, UndoFlags new_op);
35
35
 
36
36
  void WriteCatalogEntry(CatalogEntry &entry, data_ptr_t extra_data);
37
37
  void WriteDelete(DeleteInfo &info);
@@ -1,3 +1,4 @@
1
+ #include "duckdb/main/attached_database.hpp"
1
2
  #include "duckdb/logging/file_system_logger.hpp"
2
3
  #include "duckdb/logging/log_type.hpp"
3
4
  #include "duckdb/common/file_opener.hpp"
@@ -13,7 +14,18 @@ constexpr LogLevel FileSystemLogType::LEVEL;
13
14
  constexpr LogLevel QueryLogType::LEVEL;
14
15
  constexpr LogLevel HTTPLogType::LEVEL;
15
16
  constexpr LogLevel PhysicalOperatorLogType::LEVEL;
17
+ constexpr LogLevel CheckpointLogType::LEVEL;
16
18
 
19
+ //===--------------------------------------------------------------------===//
20
+ // QueryLogType
21
+ //===--------------------------------------------------------------------===//
22
+ string QueryLogType::ConstructLogMessage(const string &str) {
23
+ return str;
24
+ }
25
+
26
+ //===--------------------------------------------------------------------===//
27
+ // FileSystemLogType
28
+ //===--------------------------------------------------------------------===//
17
29
  FileSystemLogType::FileSystemLogType() : LogType(NAME, LEVEL, GetLogType()) {
18
30
  }
19
31
 
@@ -36,6 +48,9 @@ LogicalType FileSystemLogType::GetLogType() {
36
48
  return LogicalType::STRUCT(child_list);
37
49
  }
38
50
 
51
+ //===--------------------------------------------------------------------===//
52
+ // HTTPLogType
53
+ //===--------------------------------------------------------------------===//
39
54
  HTTPLogType::HTTPLogType() : LogType(NAME, LEVEL, GetLogType()) {
40
55
  }
41
56
 
@@ -92,6 +107,9 @@ string HTTPLogType::ConstructLogMessage(BaseRequest &request, optional_ptr<HTTPR
92
107
  return Value::STRUCT(child_list).ToString();
93
108
  }
94
109
 
110
+ //===--------------------------------------------------------------------===//
111
+ // PhysicalOperatorLogType
112
+ //===--------------------------------------------------------------------===//
95
113
  PhysicalOperatorLogType::PhysicalOperatorLogType() : LogType(NAME, LEVEL, GetLogType()) {
96
114
  }
97
115
 
@@ -129,5 +147,50 @@ string PhysicalOperatorLogType::ConstructLogMessage(const PhysicalOperator &phys
129
147
 
130
148
  return Value::STRUCT(std::move(child_list)).ToString();
131
149
  }
150
+ //===--------------------------------------------------------------------===//
151
+ // CheckpointLogType
152
+ //===--------------------------------------------------------------------===//
153
+ CheckpointLogType::CheckpointLogType() : LogType(NAME, LEVEL, GetLogType()) {
154
+ }
155
+
156
+ LogicalType CheckpointLogType::GetLogType() {
157
+ child_list_t<LogicalType> child_list = {
158
+ {"database", LogicalType::VARCHAR},
159
+ {"schema", LogicalType::VARCHAR},
160
+ {"table", LogicalType::VARCHAR},
161
+ {"type", LogicalType::VARCHAR},
162
+ {"info", LogicalType::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR)},
163
+ };
164
+ return LogicalType::STRUCT(child_list);
165
+ }
166
+
167
+ string CheckpointLogType::CreateLog(const AttachedDatabase &db, DataTableInfo &table, const char *op_name,
168
+ vector<Value> map_keys, vector<Value> map_values) {
169
+ child_list_t<Value> child_list = {
170
+ {"database", db.name},
171
+ {"schema", table.GetSchemaName()},
172
+ {"table", table.GetTableName()},
173
+ {"type", op_name},
174
+ {"info", Value::MAP(LogicalType::VARCHAR, LogicalType::VARCHAR, std::move(map_keys), std::move(map_values))},
175
+ };
176
+
177
+ return Value::STRUCT(std::move(child_list)).ToString();
178
+ }
179
+
180
+ string CheckpointLogType::ConstructLogMessage(const AttachedDatabase &db, DataTableInfo &table, idx_t segment_idx,
181
+ idx_t merge_count, idx_t target_count, idx_t merge_rows,
182
+ idx_t row_start) {
183
+ vector<Value> map_keys = {"segment_idx", "merge_count", "target_count", "merge_rows", "row_start"};
184
+ vector<Value> map_values = {to_string(segment_idx), to_string(merge_count), to_string(target_count),
185
+ to_string(merge_rows), to_string(row_start)};
186
+ return CreateLog(db, table, "vacuum", std::move(map_keys), std::move(map_values));
187
+ }
188
+
189
+ string CheckpointLogType::ConstructLogMessage(const AttachedDatabase &db, DataTableInfo &table, idx_t segment_idx,
190
+ RowGroup &row_group) {
191
+ vector<Value> map_keys = {"segment_idx", "start", "count"};
192
+ vector<Value> map_values = {to_string(segment_idx), to_string(row_group.start), to_string(row_group.count.load())};
193
+ return CreateLog(db, table, "checkpoint", std::move(map_keys), std::move(map_values));
194
+ }
132
195
 
133
196
  } // namespace duckdb
@@ -22,6 +22,10 @@ StoredDatabasePath::~StoredDatabasePath() {
22
22
  manager.EraseDatabasePath(path);
23
23
  }
24
24
 
25
+ void StoredDatabasePath::OnDetach() {
26
+ manager.DetachDatabase(path);
27
+ }
28
+
25
29
  //===--------------------------------------------------------------------===//
26
30
  // Attach Options
27
31
  //===--------------------------------------------------------------------===//
@@ -155,6 +159,13 @@ bool AttachedDatabase::NameIsReserved(const string &name) {
155
159
  return name == DEFAULT_SCHEMA || name == TEMP_CATALOG || name == SYSTEM_CATALOG;
156
160
  }
157
161
 
162
+ string AttachedDatabase::StoredPath() const {
163
+ if (stored_database_path) {
164
+ return stored_database_path->path;
165
+ }
166
+ return string();
167
+ }
168
+
158
169
  static string RemoveQueryParams(const string &name) {
159
170
  auto vec = StringUtil::Split(name, "?");
160
171
  D_ASSERT(!vec.empty());
@@ -227,10 +238,12 @@ void AttachedDatabase::SetReadOnlyDatabase() {
227
238
  }
228
239
 
229
240
  void AttachedDatabase::OnDetach(ClientContext &context) {
230
- if (!catalog) {
231
- return;
241
+ if (catalog) {
242
+ catalog->OnDetach(context);
243
+ }
244
+ if (stored_database_path) {
245
+ stored_database_path->OnDetach();
232
246
  }
233
- catalog->OnDetach(context);
234
247
  }
235
248
 
236
249
  void AttachedDatabase::Close() {
@@ -217,15 +217,15 @@ void ClientContext::BeginQueryInternal(ClientContextLock &lock, const string &qu
217
217
  state->QueryBegin(*this);
218
218
  }
219
219
 
220
- // Flush the old Logger
220
+ // Flush the old logger.
221
221
  logger->Flush();
222
222
 
223
- // Refresh the logger to ensure we are in sync with global log settings
224
- LoggingContext context(LogContextScope::CONNECTION);
225
- context.connection_id = connection_id;
226
- context.transaction_id = transaction.ActiveTransaction().global_transaction_id;
227
- context.query_id = transaction.GetActiveQuery();
228
- logger = db->GetLogManager().CreateLogger(context, true);
223
+ // Refresh the logger to ensure we are in sync with the global log settings.
224
+ LoggingContext logging_context(LogContextScope::CONNECTION);
225
+ logging_context.connection_id = connection_id;
226
+ logging_context.transaction_id = transaction.ActiveTransaction().global_transaction_id;
227
+ logging_context.query_id = transaction.GetActiveQuery();
228
+ logger = db->GetLogManager().CreateLogger(logging_context, true);
229
229
  DUCKDB_LOG(*this, QueryLogType, query);
230
230
  }
231
231
 
@@ -889,6 +889,10 @@ unique_ptr<PendingQueryResult> ClientContext::PendingStatementOrPreparedStatemen
889
889
  shared_ptr<PreparedStatementData> &prepared, const PendingQueryParameters &parameters) {
890
890
  unique_ptr<PendingQueryResult> pending;
891
891
 
892
+ // Start the profiler.
893
+ auto &profiler = QueryProfiler::Get(*this);
894
+ profiler.StartQuery(query, IsExplainAnalyze(statement ? statement.get() : prepared->unbound_statement.get()));
895
+
892
896
  try {
893
897
  BeginQueryInternal(lock, query);
894
898
  } catch (std::exception &ex) {
@@ -900,9 +904,6 @@ unique_ptr<PendingQueryResult> ClientContext::PendingStatementOrPreparedStatemen
900
904
  }
901
905
  return ErrorResult<PendingQueryResult>(std::move(error), query);
902
906
  }
903
- // start the profiler
904
- auto &profiler = QueryProfiler::Get(*this);
905
- profiler.StartQuery(query, IsExplainAnalyze(statement ? statement.get() : prepared->unbound_statement.get()));
906
907
 
907
908
  bool invalidate_query = true;
908
909
  try {
@@ -1406,15 +1407,7 @@ unique_ptr<QueryResult> ClientContext::Execute(const shared_ptr<Relation> &relat
1406
1407
  return ErrorResult<MaterializedQueryResult>(ErrorData(err_str));
1407
1408
  }
1408
1409
 
1409
- SettingLookupResult ClientContext::TryGetCurrentSetting(const std::string &key, Value &result) const {
1410
- // first check the built-in settings
1411
- auto &db_config = DBConfig::GetConfig(*this);
1412
- auto option = db_config.GetOptionByName(key);
1413
- if (option && option->get_setting) {
1414
- result = option->get_setting(*this);
1415
- return SettingLookupResult(SettingScope::LOCAL);
1416
- }
1417
-
1410
+ SettingLookupResult ClientContext::TryGetCurrentSettingInternal(const string &key, Value &result) const {
1418
1411
  // check the client session values
1419
1412
  const auto &session_config_map = config.set_variables;
1420
1413
 
@@ -1428,6 +1421,21 @@ SettingLookupResult ClientContext::TryGetCurrentSetting(const std::string &key,
1428
1421
  return db->TryGetCurrentSetting(key, result);
1429
1422
  }
1430
1423
 
1424
+ SettingLookupResult ClientContext::TryGetCurrentSetting(const string &key, Value &result) const {
1425
+ // first check the built-in settings
1426
+ auto &db_config = DBConfig::GetConfig(*this);
1427
+ auto option = db_config.GetOptionByName(key);
1428
+ if (option) {
1429
+ if (option->get_setting) {
1430
+ result = option->get_setting(*this);
1431
+ return SettingLookupResult(SettingScope::LOCAL);
1432
+ }
1433
+ // alias - search for the default key
1434
+ return TryGetCurrentSettingInternal(option->name, result);
1435
+ }
1436
+ return TryGetCurrentSettingInternal(key, result);
1437
+ }
1438
+
1431
1439
  ParserOptions ClientContext::GetParserOptions() const {
1432
1440
  auto &client_config = ClientConfig::GetConfig(*this);
1433
1441
  ParserOptions options;
@@ -19,7 +19,7 @@
19
19
  namespace duckdb {
20
20
 
21
21
  Connection::Connection(DatabaseInstance &database)
22
- : context(make_shared_ptr<ClientContext>(database.shared_from_this())), warning_cb(nullptr) {
22
+ : context(make_shared_ptr<ClientContext>(database.shared_from_this())) {
23
23
  auto &connection_manager = ConnectionManager::Get(database);
24
24
  connection_manager.AddConnection(*context);
25
25
  connection_manager.AssignConnectionId(*this);
@@ -31,18 +31,15 @@ Connection::Connection(DatabaseInstance &database)
31
31
  }
32
32
 
33
33
  Connection::Connection(DuckDB &database) : Connection(*database.instance) {
34
- // Initialization of warning_cb happens in the other constructor
35
34
  }
36
35
 
37
- Connection::Connection(Connection &&other) noexcept : warning_cb(nullptr) {
36
+ Connection::Connection(Connection &&other) noexcept {
38
37
  std::swap(context, other.context);
39
- std::swap(warning_cb, other.warning_cb);
40
38
  std::swap(connection_id, other.connection_id);
41
39
  }
42
40
 
43
41
  Connection &Connection::operator=(Connection &&other) noexcept {
44
42
  std::swap(context, other.context);
45
- std::swap(warning_cb, other.warning_cb);
46
43
  std::swap(connection_id, other.connection_id);
47
44
  return *this;
48
45
  }
@@ -7,7 +7,7 @@ namespace duckdb {
7
7
 
8
8
  idx_t DatabaseFilePathManager::ApproxDatabaseCount() const {
9
9
  lock_guard<mutex> path_lock(db_paths_lock);
10
- return db_paths_to_name.size();
10
+ return db_paths.size();
11
11
  }
12
12
 
13
13
  InsertDatabasePathResult DatabaseFilePathManager::InsertDatabasePath(const string &path, const string &name,
@@ -18,14 +18,20 @@ InsertDatabasePathResult DatabaseFilePathManager::InsertDatabasePath(const strin
18
18
  }
19
19
 
20
20
  lock_guard<mutex> path_lock(db_paths_lock);
21
- auto entry = db_paths_to_name.emplace(path, name);
21
+ auto entry = db_paths.emplace(path, DatabasePathInfo(name));
22
22
  if (!entry.second) {
23
- if (on_conflict == OnCreateConflict::IGNORE_ON_CONFLICT && entry.first->second == name) {
24
- return InsertDatabasePathResult::ALREADY_EXISTS;
23
+ auto &existing = entry.first->second;
24
+ if (on_conflict == OnCreateConflict::IGNORE_ON_CONFLICT && existing.name == name) {
25
+ if (existing.is_attached) {
26
+ return InsertDatabasePathResult::ALREADY_EXISTS;
27
+ }
28
+ throw BinderException("Unique file handle conflict: Cannot attach \"%s\" - the database file \"%s\" is in "
29
+ "the process of being detached",
30
+ name, path);
25
31
  }
26
32
  throw BinderException("Unique file handle conflict: Cannot attach \"%s\" - the database file \"%s\" is already "
27
33
  "attached by database \"%s\"",
28
- name, path, entry.first->second);
34
+ name, path, existing.name);
29
35
  }
30
36
  options.stored_database_path = make_uniq<StoredDatabasePath>(*this, path, name);
31
37
  return InsertDatabasePathResult::SUCCESS;
@@ -36,7 +42,18 @@ void DatabaseFilePathManager::EraseDatabasePath(const string &path) {
36
42
  return;
37
43
  }
38
44
  lock_guard<mutex> path_lock(db_paths_lock);
39
- db_paths_to_name.erase(path);
45
+ db_paths.erase(path);
46
+ }
47
+
48
+ void DatabaseFilePathManager::DetachDatabase(const string &path) {
49
+ if (path.empty() || path == IN_MEMORY_PATH) {
50
+ return;
51
+ }
52
+ lock_guard<mutex> path_lock(db_paths_lock);
53
+ auto entry = db_paths.find(path);
54
+ if (entry != db_paths.end()) {
55
+ entry->second.is_attached = false;
56
+ }
40
57
  }
41
58
 
42
59
  } // namespace duckdb
@@ -52,18 +52,33 @@ optional_ptr<AttachedDatabase> DatabaseManager::GetDatabase(ClientContext &conte
52
52
  return database;
53
53
  }
54
54
  lock_guard<mutex> guard(databases_lock);
55
+ shared_ptr<AttachedDatabase> db;
55
56
  if (StringUtil::Lower(name) == TEMP_CATALOG) {
56
- return meta_transaction.UseDatabase(context.client_data->temporary_objects);
57
+ db = context.client_data->temporary_objects;
58
+ } else {
59
+ db = GetDatabaseInternal(guard, name);
57
60
  }
61
+ if (!db) {
62
+ return nullptr;
63
+ }
64
+ return meta_transaction.UseDatabase(db);
65
+ }
66
+
67
+ shared_ptr<AttachedDatabase> DatabaseManager::GetDatabase(const string &name) {
68
+ lock_guard<mutex> guard(databases_lock);
69
+ return GetDatabaseInternal(guard, name);
70
+ }
71
+
72
+ shared_ptr<AttachedDatabase> DatabaseManager::GetDatabaseInternal(const lock_guard<mutex> &, const string &name) {
58
73
  if (StringUtil::Lower(name) == SYSTEM_CATALOG) {
59
- return meta_transaction.UseDatabase(system);
74
+ return system;
60
75
  }
61
76
  auto entry = databases.find(name);
62
77
  if (entry == databases.end()) {
63
78
  // not found
64
79
  return nullptr;
65
80
  }
66
- return meta_transaction.UseDatabase(entry->second);
81
+ return entry->second;
67
82
  }
68
83
 
69
84
  shared_ptr<AttachedDatabase> DatabaseManager::AttachDatabase(ClientContext &context, AttachInfo &info,
@@ -367,7 +367,9 @@ HTTPUtil::RunRequestWithRetry(const std::function<unique_ptr<HTTPResponse>(void)
367
367
 
368
368
  try {
369
369
  response = on_request();
370
- response->url = request.url;
370
+ if (response) {
371
+ response->url = request.url;
372
+ }
371
373
  } catch (IOException &e) {
372
374
  exception_error = e.what();
373
375
  caught_e = std::current_exception();
@@ -29,6 +29,14 @@ BoundStatement CreateTableRelation::Bind(Binder &binder) {
29
29
  return binder.Bind(stmt.Cast<SQLStatement>());
30
30
  }
31
31
 
32
+ unique_ptr<QueryNode> CreateTableRelation::GetQueryNode() {
33
+ throw InternalException("Cannot create a query node from a create table relation");
34
+ }
35
+
36
+ string CreateTableRelation::GetQuery() {
37
+ return string();
38
+ }
39
+
32
40
  const vector<ColumnDefinition> &CreateTableRelation::Columns() {
33
41
  return columns;
34
42
  }
@@ -35,6 +35,14 @@ BoundStatement CreateViewRelation::Bind(Binder &binder) {
35
35
  return binder.Bind(stmt.Cast<SQLStatement>());
36
36
  }
37
37
 
38
+ unique_ptr<QueryNode> CreateViewRelation::GetQueryNode() {
39
+ throw InternalException("Cannot create a query node from an update relation");
40
+ }
41
+
42
+ string CreateViewRelation::GetQuery() {
43
+ return string();
44
+ }
45
+
38
46
  const vector<ColumnDefinition> &CreateViewRelation::Columns() {
39
47
  return columns;
40
48
  }
@@ -26,6 +26,14 @@ BoundStatement DeleteRelation::Bind(Binder &binder) {
26
26
  return binder.Bind(stmt.Cast<SQLStatement>());
27
27
  }
28
28
 
29
+ unique_ptr<QueryNode> DeleteRelation::GetQueryNode() {
30
+ throw InternalException("Cannot create a query node from a delete relation");
31
+ }
32
+
33
+ string DeleteRelation::GetQuery() {
34
+ return string();
35
+ }
36
+
29
37
  const vector<ColumnDefinition> &DeleteRelation::Columns() {
30
38
  return columns;
31
39
  }
@@ -20,6 +20,14 @@ BoundStatement ExplainRelation::Bind(Binder &binder) {
20
20
  return binder.Bind(explain.Cast<SQLStatement>());
21
21
  }
22
22
 
23
+ unique_ptr<QueryNode> ExplainRelation::GetQueryNode() {
24
+ throw InternalException("Cannot create a query node from an explain relation");
25
+ }
26
+
27
+ string ExplainRelation::GetQuery() {
28
+ return string();
29
+ }
30
+
23
31
  const vector<ColumnDefinition> &ExplainRelation::Columns() {
24
32
  return columns;
25
33
  }