duckdb 1.2.1-dev6.0 → 1.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (117) hide show
  1. package/package.json +1 -1
  2. package/src/duckdb/extension/core_functions/aggregate/distributive/string_agg.cpp +14 -22
  3. package/src/duckdb/extension/core_functions/aggregate/nested/list.cpp +0 -1
  4. package/src/duckdb/extension/core_functions/lambda_functions.cpp +0 -11
  5. package/src/duckdb/extension/core_functions/scalar/list/list_aggregates.cpp +18 -6
  6. package/src/duckdb/extension/icu/icu-datefunc.cpp +9 -2
  7. package/src/duckdb/extension/icu/icu-strptime.cpp +7 -11
  8. package/src/duckdb/extension/icu/include/icu-datefunc.hpp +3 -1
  9. package/src/duckdb/extension/json/buffered_json_reader.cpp +18 -31
  10. package/src/duckdb/extension/json/json_extension.cpp +8 -3
  11. package/src/duckdb/extension/parquet/column_reader.cpp +4 -6
  12. package/src/duckdb/extension/parquet/column_writer.cpp +33 -12
  13. package/src/duckdb/extension/parquet/include/column_reader.hpp +0 -2
  14. package/src/duckdb/extension/parquet/include/parquet_bss_encoder.hpp +0 -1
  15. package/src/duckdb/extension/parquet/include/parquet_dlba_encoder.hpp +1 -2
  16. package/src/duckdb/src/catalog/catalog.cpp +12 -0
  17. package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +1 -1
  18. package/src/duckdb/src/catalog/catalog_entry_retriever.cpp +1 -1
  19. package/src/duckdb/src/catalog/catalog_search_path.cpp +8 -8
  20. package/src/duckdb/src/common/bind_helpers.cpp +3 -0
  21. package/src/duckdb/src/common/compressed_file_system.cpp +2 -0
  22. package/src/duckdb/src/common/hive_partitioning.cpp +1 -1
  23. package/src/duckdb/src/common/multi_file_reader.cpp +3 -3
  24. package/src/duckdb/src/execution/aggregate_hashtable.cpp +1 -1
  25. package/src/duckdb/src/execution/index/art/art.cpp +19 -6
  26. package/src/duckdb/src/execution/index/art/iterator.cpp +7 -3
  27. package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +11 -4
  28. package/src/duckdb/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp +2 -2
  29. package/src/duckdb/src/execution/operator/csv_scanner/encode/csv_encoder.cpp +5 -1
  30. package/src/duckdb/src/execution/operator/csv_scanner/scanner/base_scanner.cpp +3 -2
  31. package/src/duckdb/src/execution/operator/csv_scanner/scanner/csv_schema.cpp +2 -2
  32. package/src/duckdb/src/execution/operator/csv_scanner/scanner/scanner_boundary.cpp +1 -1
  33. package/src/duckdb/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +20 -12
  34. package/src/duckdb/src/execution/operator/csv_scanner/sniffer/dialect_detection.cpp +19 -22
  35. package/src/duckdb/src/execution/operator/csv_scanner/sniffer/type_refinement.cpp +1 -1
  36. package/src/duckdb/src/execution/operator/csv_scanner/util/csv_error.cpp +1 -0
  37. package/src/duckdb/src/execution/operator/csv_scanner/util/csv_reader_options.cpp +16 -0
  38. package/src/duckdb/src/execution/operator/helper/physical_reservoir_sample.cpp +1 -0
  39. package/src/duckdb/src/execution/operator/helper/physical_streaming_sample.cpp +16 -7
  40. package/src/duckdb/src/execution/operator/persistent/physical_batch_insert.cpp +3 -1
  41. package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp +11 -1
  42. package/src/duckdb/src/execution/operator/schema/physical_create_art_index.cpp +5 -7
  43. package/src/duckdb/src/execution/physical_plan/plan_create_index.cpp +11 -0
  44. package/src/duckdb/src/execution/physical_plan/plan_sample.cpp +1 -3
  45. package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp +14 -5
  46. package/src/duckdb/src/execution/sample/reservoir_sample.cpp +24 -12
  47. package/src/duckdb/src/function/scalar/generic/getvariable.cpp +3 -3
  48. package/src/duckdb/src/function/table/version/pragma_version.cpp +3 -3
  49. package/src/duckdb/src/function/window/window_aggregate_states.cpp +3 -0
  50. package/src/duckdb/src/function/window/window_boundaries_state.cpp +108 -48
  51. package/src/duckdb/src/function/window/window_constant_aggregator.cpp +5 -5
  52. package/src/duckdb/src/function/window/window_distinct_aggregator.cpp +6 -0
  53. package/src/duckdb/src/include/duckdb/catalog/catalog_entry_retriever.hpp +1 -1
  54. package/src/duckdb/src/include/duckdb/catalog/catalog_search_path.hpp +10 -9
  55. package/src/duckdb/src/include/duckdb/common/adbc/adbc-init.hpp +1 -1
  56. package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp +2 -2
  57. package/src/duckdb/src/include/duckdb/execution/index/art/iterator.hpp +2 -0
  58. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/base_scanner.hpp +1 -1
  59. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_buffer.hpp +5 -4
  60. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_option.hpp +1 -1
  61. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_schema.hpp +2 -2
  62. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/encode/csv_encoder.hpp +1 -1
  63. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/sniffer/csv_sniffer.hpp +1 -1
  64. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp +2 -2
  65. package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_streaming_sample.hpp +3 -7
  66. package/src/duckdb/src/include/duckdb/execution/reservoir_sample.hpp +2 -1
  67. package/src/duckdb/src/include/duckdb/function/lambda_functions.hpp +11 -3
  68. package/src/duckdb/src/include/duckdb/function/window/window_boundaries_state.hpp +4 -0
  69. package/src/duckdb/src/include/duckdb/main/client_context_state.hpp +4 -0
  70. package/src/duckdb/src/include/duckdb/main/extension_entries.hpp +25 -7
  71. package/src/duckdb/src/include/duckdb/main/pending_query_result.hpp +2 -0
  72. package/src/duckdb/src/include/duckdb/main/query_profiler.hpp +7 -0
  73. package/src/duckdb/src/include/duckdb/optimizer/filter_combiner.hpp +2 -2
  74. package/src/duckdb/src/include/duckdb/optimizer/late_materialization.hpp +2 -1
  75. package/src/duckdb/src/include/duckdb/optimizer/optimizer_extension.hpp +11 -5
  76. package/src/duckdb/src/include/duckdb/parallel/executor_task.hpp +4 -1
  77. package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp +0 -1
  78. package/src/duckdb/src/include/duckdb/parallel/task_executor.hpp +3 -0
  79. package/src/duckdb/src/include/duckdb/parallel/task_notifier.hpp +27 -0
  80. package/src/duckdb/src/include/duckdb/parallel/task_scheduler.hpp +4 -0
  81. package/src/duckdb/src/include/duckdb/planner/expression/bound_subquery_expression.hpp +1 -1
  82. package/src/duckdb/src/include/duckdb/planner/tableref/bound_cteref.hpp +1 -0
  83. package/src/duckdb/src/include/duckdb/storage/checkpoint/table_data_writer.hpp +3 -1
  84. package/src/duckdb/src/include/duckdb/storage/checkpoint_manager.hpp +7 -1
  85. package/src/duckdb/src/include/duckdb/storage/storage_manager.hpp +3 -2
  86. package/src/duckdb/src/include/duckdb.h +495 -480
  87. package/src/duckdb/src/main/attached_database.cpp +1 -1
  88. package/src/duckdb/src/main/capi/duckdb-c.cpp +5 -1
  89. package/src/duckdb/src/main/capi/helper-c.cpp +8 -0
  90. package/src/duckdb/src/main/config.cpp +7 -1
  91. package/src/duckdb/src/main/database.cpp +8 -8
  92. package/src/duckdb/src/main/extension/extension_helper.cpp +3 -1
  93. package/src/duckdb/src/main/extension/extension_load.cpp +12 -12
  94. package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp +1 -0
  95. package/src/duckdb/src/optimizer/join_order/query_graph_manager.cpp +2 -2
  96. package/src/duckdb/src/optimizer/late_materialization.cpp +26 -5
  97. package/src/duckdb/src/optimizer/optimizer.cpp +12 -1
  98. package/src/duckdb/src/parallel/executor_task.cpp +10 -6
  99. package/src/duckdb/src/parallel/task_executor.cpp +4 -1
  100. package/src/duckdb/src/parallel/task_notifier.cpp +23 -0
  101. package/src/duckdb/src/parallel/task_scheduler.cpp +33 -0
  102. package/src/duckdb/src/parser/transform/expression/transform_subquery.cpp +4 -1
  103. package/src/duckdb/src/planner/binder/expression/bind_subquery_expression.cpp +1 -1
  104. package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp +4 -2
  105. package/src/duckdb/src/planner/binder/statement/bind_create.cpp +7 -2
  106. package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +6 -5
  107. package/src/duckdb/src/storage/checkpoint/table_data_writer.cpp +4 -2
  108. package/src/duckdb/src/storage/checkpoint_manager.cpp +4 -3
  109. package/src/duckdb/src/storage/compression/string_uncompressed.cpp +21 -10
  110. package/src/duckdb/src/storage/storage_info.cpp +2 -0
  111. package/src/duckdb/src/storage/storage_manager.cpp +2 -2
  112. package/src/duckdb/src/storage/table/row_group.cpp +5 -6
  113. package/src/duckdb/src/storage/table/scan_state.cpp +6 -0
  114. package/src/duckdb/src/transaction/duck_transaction.cpp +11 -3
  115. package/src/duckdb/src/transaction/duck_transaction_manager.cpp +2 -2
  116. package/src/duckdb/third_party/concurrentqueue/concurrentqueue.h +17 -0
  117. package/src/duckdb/ub_src_parallel.cpp +2 -0
@@ -48,20 +48,21 @@ public:
48
48
  DUCKDB_API void Set(vector<CatalogSearchEntry> new_paths, CatalogSetPathType set_type);
49
49
  DUCKDB_API void Reset();
50
50
 
51
- DUCKDB_API const vector<CatalogSearchEntry> &Get();
52
- const vector<CatalogSearchEntry> &GetSetPaths() {
51
+ DUCKDB_API const vector<CatalogSearchEntry> &Get() const;
52
+ const vector<CatalogSearchEntry> &GetSetPaths() const {
53
53
  return set_paths;
54
54
  }
55
- DUCKDB_API const CatalogSearchEntry &GetDefault();
55
+ DUCKDB_API const CatalogSearchEntry &GetDefault() const;
56
56
  //! FIXME: this method is deprecated
57
- DUCKDB_API string GetDefaultSchema(const string &catalog);
58
- DUCKDB_API string GetDefaultSchema(ClientContext &context, const string &catalog);
59
- DUCKDB_API string GetDefaultCatalog(const string &schema);
57
+ DUCKDB_API string GetDefaultSchema(const string &catalog) const;
58
+ DUCKDB_API string GetDefaultSchema(ClientContext &context, const string &catalog) const;
59
+ DUCKDB_API string GetDefaultCatalog(const string &schema) const;
60
60
 
61
- DUCKDB_API vector<string> GetSchemasForCatalog(const string &catalog);
62
- DUCKDB_API vector<string> GetCatalogsForSchema(const string &schema);
61
+ DUCKDB_API vector<string> GetSchemasForCatalog(const string &catalog) const;
62
+ DUCKDB_API vector<string> GetCatalogsForSchema(const string &schema) const;
63
63
 
64
- DUCKDB_API bool SchemaInSearchPath(ClientContext &context, const string &catalog_name, const string &schema_name);
64
+ DUCKDB_API bool SchemaInSearchPath(ClientContext &context, const string &catalog_name,
65
+ const string &schema_name) const;
65
66
 
66
67
  private:
67
68
  //! Set paths without checking if they exist
@@ -30,7 +30,7 @@ extern "C" {
30
30
  typedef uint8_t AdbcStatusCode;
31
31
 
32
32
  //! We gotta leak the symbols of the init function
33
- DUCKDB_API AdbcStatusCode duckdb_adbc_init(int version, void *driver, struct AdbcError *error);
33
+ DUCKDB_C_API AdbcStatusCode duckdb_adbc_init(int version, void *driver, struct AdbcError *error);
34
34
 
35
35
  #ifdef __cplusplus
36
36
  }
@@ -168,7 +168,7 @@ struct MultiFileFilterEntry {
168
168
  struct MultiFileConstantEntry {
169
169
  MultiFileConstantEntry(idx_t column_id, Value value_p) : column_id(column_id), value(std::move(value_p)) {
170
170
  }
171
- //! The column id to apply the constant value to
171
+ //! The (global) column id to apply the constant value to
172
172
  idx_t column_id;
173
173
  //! The constant value
174
174
  Value value;
@@ -273,7 +273,7 @@ struct MultiFileReader {
273
273
  const string &initial_file, const MultiFileReaderBindData &options,
274
274
  optional_ptr<MultiFileReaderGlobalState> global_state);
275
275
  //! Populated the filter_map
276
- DUCKDB_API virtual void CreateFilterMap(const vector<MultiFileReaderColumnDefinition> &global_columns,
276
+ DUCKDB_API virtual void CreateFilterMap(const vector<ColumnIndex> &global_column_ids,
277
277
  optional_ptr<TableFilterSet> filters, MultiFileReaderData &reader_data,
278
278
  optional_ptr<MultiFileReaderGlobalState> global_state);
279
279
 
@@ -90,6 +90,8 @@ private:
90
90
  GateStatus status;
91
91
  //! Depth in a nested leaf.
92
92
  uint8_t nested_depth = 0;
93
+ //! True, if we entered a nested leaf to retrieve the next node.
94
+ bool entered_nested_leaf = false;
93
95
 
94
96
  private:
95
97
  //! Goes to the next leaf in the ART and sets it as last_leaf,
@@ -116,7 +116,7 @@ class BaseScanner {
116
116
  public:
117
117
  explicit BaseScanner(shared_ptr<CSVBufferManager> buffer_manager, shared_ptr<CSVStateMachine> state_machine,
118
118
  shared_ptr<CSVErrorHandler> error_handler, bool sniffing = false,
119
- shared_ptr<CSVFileScan> csv_file_scan = nullptr, CSVIterator iterator = {});
119
+ shared_ptr<CSVFileScan> csv_file_scan = nullptr, const CSVIterator &iterator = {});
120
120
 
121
121
  virtual ~BaseScanner() = default;
122
122
 
@@ -45,14 +45,15 @@ class CSVBuffer {
45
45
  public:
46
46
  //! Constructor for Initial Buffer
47
47
  CSVBuffer(ClientContext &context, idx_t buffer_size_p, CSVFileHandle &file_handle,
48
- idx_t &global_csv_current_position, idx_t file_number);
48
+ const idx_t &global_csv_current_position, idx_t file_number);
49
49
 
50
50
  //! Constructor for `Next()` Buffers
51
51
  CSVBuffer(CSVFileHandle &file_handle, ClientContext &context, idx_t buffer_size, idx_t global_csv_current_position,
52
52
  idx_t file_number_p, idx_t buffer_idx);
53
53
 
54
54
  //! Creates a new buffer with the next part of the CSV File
55
- shared_ptr<CSVBuffer> Next(CSVFileHandle &file_handle, idx_t buffer_size, idx_t file_number, bool &has_seaked);
55
+ shared_ptr<CSVBuffer> Next(CSVFileHandle &file_handle, idx_t buffer_size, idx_t file_number,
56
+ bool &has_seaked) const;
56
57
 
57
58
  //! Gets the buffer actual size
58
59
  idx_t GetBufferSize() const;
@@ -67,12 +68,12 @@ public:
67
68
  //! Wrapper for the Pin Function, if it can seek, it means that the buffer might have been destroyed, hence we must
68
69
  //! Scan it from the disk file again.
69
70
  shared_ptr<CSVBufferHandle> Pin(CSVFileHandle &file_handle, bool &has_seeked);
70
- //! Wrapper for the unpin
71
+ //! Wrapper for unpin
71
72
  void Unpin();
72
73
  char *Ptr() {
73
74
  return char_ptr_cast(handle.Ptr());
74
75
  }
75
- bool IsUnloaded() {
76
+ bool IsUnloaded() const {
76
77
  return block->IsUnloaded();
77
78
  }
78
79
 
@@ -30,7 +30,7 @@ struct CSVOption { // NOLINT: work-around bug in clang-tidy
30
30
  public:
31
31
  CSVOption(T value_p) : value(value_p) { // NOLINT: allow implicit conversion from value
32
32
  }
33
- CSVOption(T value_p, bool set_by_user_p) : value(value_p), set_by_user(set_by_user_p) {
33
+ CSVOption(T value_p, bool set_by_user_p) : set_by_user(set_by_user_p), value(value_p) {
34
34
  }
35
35
 
36
36
  CSVOption() {};
@@ -14,7 +14,7 @@
14
14
  namespace duckdb {
15
15
  //! Basic CSV Column Info
16
16
  struct CSVColumnInfo {
17
- CSVColumnInfo(string &name_p, LogicalType &type_p) : name(name_p), type(type_p) {
17
+ CSVColumnInfo(const string &name_p, const LogicalType &type_p) : name(name_p), type(type_p) {
18
18
  }
19
19
  string name;
20
20
  LogicalType type;
@@ -25,7 +25,7 @@ struct CSVSchema {
25
25
  explicit CSVSchema(const bool empty = false) : empty(empty) {
26
26
  }
27
27
 
28
- CSVSchema(vector<string> &names, vector<LogicalType> &types, const string &file_path, idx_t rows_read,
28
+ CSVSchema(const vector<string> &names, const vector<LogicalType> &types, const string &file_path, idx_t rows_read,
29
29
  const bool empty = false);
30
30
 
31
31
  //! Initializes the schema based on names and types
@@ -46,7 +46,7 @@ private:
46
46
  class CSVEncoder {
47
47
  public:
48
48
  //! Constructor, basically takes an encoding and the output buffer size
49
- CSVEncoder(DBConfig &config, const string &encoding_name, idx_t buffer_size);
49
+ CSVEncoder(const DBConfig &config, const string &encoding_name, idx_t buffer_size);
50
50
  //! Main encode function, it reads the file into an encoded buffer and converts it to the output buffer
51
51
  idx_t Encode(FileHandle &file_handle_input, char *output_buffer, const idx_t decoded_buffer_size);
52
52
  string encoding_name;
@@ -222,7 +222,7 @@ private:
222
222
  //! ------------------ Type Refinement ------------------ //
223
223
  //! ------------------------------------------------------//
224
224
  void RefineTypes();
225
- bool TryCastVector(Vector &parse_chunk_col, idx_t size, const LogicalType &sql_type);
225
+ bool TryCastVector(Vector &parse_chunk_col, idx_t size, const LogicalType &sql_type) const;
226
226
  vector<LogicalType> detected_types;
227
227
  //! If when finding a SQLNULL type in type detection we default it to varchar
228
228
  const bool default_null_to_varchar;
@@ -121,8 +121,8 @@ private:
121
121
  };
122
122
 
123
123
  struct ParseTypeInfo {
124
- ParseTypeInfo() {};
125
- ParseTypeInfo(const LogicalType &type, bool validate_utf_8_p) : validate_utf8(validate_utf_8_p) {
124
+ ParseTypeInfo() : validate_utf8(false), type_id(), internal_type(), scale(0), width(0) {};
125
+ ParseTypeInfo(const LogicalType &type, const bool validate_utf_8_p) : validate_utf8(validate_utf_8_p) {
126
126
  type_id = type.id();
127
127
  internal_type = type.InternalType();
128
128
  if (type.id() == LogicalTypeId::DECIMAL) {
@@ -19,12 +19,10 @@ public:
19
19
  static constexpr const PhysicalOperatorType TYPE = PhysicalOperatorType::STREAMING_SAMPLE;
20
20
 
21
21
  public:
22
- PhysicalStreamingSample(vector<LogicalType> types, SampleMethod method, double percentage, int64_t seed,
23
- idx_t estimated_cardinality);
22
+ PhysicalStreamingSample(vector<LogicalType> types, unique_ptr<SampleOptions> options, idx_t estimated_cardinality);
24
23
 
25
- SampleMethod method;
24
+ unique_ptr<SampleOptions> sample_options;
26
25
  double percentage;
27
- int64_t seed;
28
26
 
29
27
  public:
30
28
  // Operator interface
@@ -32,9 +30,7 @@ public:
32
30
  OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk,
33
31
  GlobalOperatorState &gstate, OperatorState &state) const override;
34
32
 
35
- bool ParallelOperator() const override {
36
- return true;
37
- }
33
+ bool ParallelOperator() const override;
38
34
 
39
35
  InsertionOrderPreservingMap<string> ParamsToString() const override;
40
36
 
@@ -201,7 +201,8 @@ public:
201
201
  void ConvertToReservoirSample();
202
202
 
203
203
  //! Get the capactiy of the data chunk reserved for storing samples
204
- idx_t GetReservoirChunkCapacity() const;
204
+ template <typename T>
205
+ T GetReservoirChunkCapacity() const;
205
206
 
206
207
  //! If for_serialization=true then the sample_chunk is not padded with extra spaces for
207
208
  //! future sampling values
@@ -17,7 +17,7 @@
17
17
 
18
18
  namespace duckdb {
19
19
 
20
- struct ListLambdaBindData : public FunctionData {
20
+ struct ListLambdaBindData final : public FunctionData {
21
21
  public:
22
22
  ListLambdaBindData(const LogicalType &return_type, unique_ptr<Expression> lambda_expr, const bool has_index = false)
23
23
  : return_type(return_type), lambda_expr(std::move(lambda_expr)), has_index(has_index) {};
@@ -30,8 +30,16 @@ public:
30
30
  bool has_index;
31
31
 
32
32
  public:
33
- bool Equals(const FunctionData &other_p) const override;
34
- unique_ptr<FunctionData> Copy() const override;
33
+ unique_ptr<FunctionData> Copy() const override {
34
+ auto lambda_expr_copy = lambda_expr ? lambda_expr->Copy() : nullptr;
35
+ return make_uniq<ListLambdaBindData>(return_type, std::move(lambda_expr_copy), has_index);
36
+ }
37
+
38
+ bool Equals(const FunctionData &other_p) const override {
39
+ auto &other = other_p.Cast<ListLambdaBindData>();
40
+ return Expression::Equals(lambda_expr, other.lambda_expr) && return_type == other.return_type &&
41
+ has_index == other.has_index;
42
+ }
35
43
 
36
44
  //! Serializes a lambda function's bind data
37
45
  static void Serialize(Serializer &serializer, const optional_ptr<FunctionData> bind_data_p,
@@ -148,6 +148,10 @@ struct WindowBoundariesState {
148
148
  idx_t valid_end = 0;
149
149
 
150
150
  FrameBounds prev;
151
+
152
+ // Extra range cursor
153
+ optional_ptr<WindowCursor> range_lo;
154
+ unique_ptr<WindowCursor> range_hi;
151
155
  };
152
156
 
153
157
  } // namespace duckdb
@@ -80,6 +80,10 @@ public:
80
80
  }
81
81
  virtual void WriteProfilingInformation(std::ostream &ss) {
82
82
  }
83
+ virtual void OnTaskStart(ClientContext &context) {
84
+ }
85
+ virtual void OnTaskStop(ClientContext &context) {
86
+ }
83
87
 
84
88
  public:
85
89
  template <class TARGET>
@@ -489,6 +489,7 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
489
489
  {"read_xlsx", "excel", CatalogType::TABLE_FUNCTION_ENTRY},
490
490
  {"reduce", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
491
491
  {"reduce_sql_statement", "sqlsmith", CatalogType::TABLE_FUNCTION_ENTRY},
492
+ {"register_geoarrow_extensions", "spatial", CatalogType::TABLE_FUNCTION_ENTRY},
492
493
  {"regr_avgx", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY},
493
494
  {"regr_avgy", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY},
494
495
  {"regr_count", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY},
@@ -538,6 +539,7 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
538
539
  {"st_centroid", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
539
540
  {"st_collect", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
540
541
  {"st_collectionextract", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
542
+ {"st_concavehull", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
541
543
  {"st_contains", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
542
544
  {"st_containsproperly", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
543
545
  {"st_convexhull", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
@@ -588,14 +590,17 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
588
590
  {"st_isvalid", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
589
591
  {"st_length", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
590
592
  {"st_length_spheroid", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
593
+ {"st_lineinterpolatepoint", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
594
+ {"st_lineinterpolatepoints", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
591
595
  {"st_linemerge", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
592
596
  {"st_linestring2dfromwkb", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
593
- {"st_list_proj_crs", "spatial", CatalogType::TABLE_FUNCTION_ENTRY},
597
+ {"st_linesubstring", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
594
598
  {"st_m", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
595
599
  {"st_makeenvelope", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
596
600
  {"st_makeline", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
597
601
  {"st_makepolygon", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
598
602
  {"st_makevalid", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
603
+ {"st_minimumrotatedrectangle", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
599
604
  {"st_mmax", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
600
605
  {"st_mmin", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
601
606
  {"st_multi", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
@@ -634,6 +639,7 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
634
639
  {"st_transform", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
635
640
  {"st_union", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
636
641
  {"st_union_agg", "spatial", CatalogType::AGGREGATE_FUNCTION_ENTRY},
642
+ {"st_voronoidiagram", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
637
643
  {"st_within", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
638
644
  {"st_x", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
639
645
  {"st_xmax", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
@@ -645,13 +651,15 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
645
651
  {"st_zmax", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
646
652
  {"st_zmflag", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
647
653
  {"st_zmin", "spatial", CatalogType::SCALAR_FUNCTION_ENTRY},
648
- {"start_ui", "motherduck", CatalogType::TABLE_FUNCTION_ENTRY},
654
+ {"start_ui", "ui", CatalogType::TABLE_FUNCTION_ENTRY},
655
+ {"start_ui_server", "ui", CatalogType::TABLE_FUNCTION_ENTRY},
649
656
  {"starts_with", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
650
657
  {"stats", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
651
658
  {"stddev", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY},
652
659
  {"stddev_pop", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY},
653
660
  {"stddev_samp", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY},
654
661
  {"stem", "fts", CatalogType::SCALAR_FUNCTION_ENTRY},
662
+ {"stop_ui_server", "ui", CatalogType::TABLE_FUNCTION_ENTRY},
655
663
  {"string_agg", "core_functions", CatalogType::AGGREGATE_FUNCTION_ENTRY},
656
664
  {"strpos", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
657
665
  {"struct_insert", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
@@ -695,6 +703,7 @@ static constexpr ExtensionFunctionEntry EXTENSION_FUNCTIONS[] = {
695
703
  {"trunc", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
696
704
  {"txid_current", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
697
705
  {"typeof", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
706
+ {"ui_is_started", "ui", CatalogType::TABLE_FUNCTION_ENTRY},
698
707
  {"unbin", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
699
708
  {"unhex", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
700
709
  {"unicode", "core_functions", CatalogType::SCALAR_FUNCTION_ENTRY},
@@ -983,6 +992,9 @@ static constexpr ExtensionEntry EXTENSION_SETTINGS[] = {
983
992
  {"sqlite_all_varchar", "sqlite_scanner"},
984
993
  {"sqlite_debug_show_queries", "sqlite_scanner"},
985
994
  {"timezone", "icu"},
995
+ {"ui_local_port", "ui"},
996
+ {"ui_polling_interval", "ui"},
997
+ {"ui_remote_url", "ui"},
986
998
  {"unsafe_enable_version_guessing", "iceberg"},
987
999
  }; // END_OF_EXTENSION_SETTINGS
988
1000
 
@@ -1070,10 +1082,16 @@ static constexpr ExtensionEntry EXTENSION_SECRET_PROVIDERS[] = {
1070
1082
  {"mysql/config", "mysql_scanner"},
1071
1083
  {"postgres/config", "postgres_scanner"}}; // EXTENSION_SECRET_PROVIDERS
1072
1084
 
1073
- static constexpr const char *AUTOLOADABLE_EXTENSIONS[] = {
1074
- "aws", "azure", "autocomplete", "core_functions", "delta", "excel",
1075
- "fts", "httpfs", "iceberg", "inet", "icu", "json",
1076
- "motherduck", "mysql_scanner", "parquet", "sqlite_scanner", "sqlsmith", "postgres_scanner",
1077
- "tpcds", "tpch", "uc_catalog"}; // END_OF_AUTOLOADABLE_EXTENSIONS
1085
+ static constexpr const char *AUTOLOADABLE_EXTENSIONS[] = {"aws", "azure",
1086
+ "autocomplete", "core_functions",
1087
+ "delta", "excel",
1088
+ "fts", "httpfs",
1089
+ "iceberg", "inet",
1090
+ "icu", "json",
1091
+ "motherduck", "mysql_scanner",
1092
+ "parquet", "sqlite_scanner",
1093
+ "sqlsmith", "postgres_scanner",
1094
+ "tpcds", "tpch",
1095
+ "uc_catalog", "ui"}; // END_OF_AUTOLOADABLE_EXTENSIONS
1078
1096
 
1079
1097
  } // namespace duckdb
@@ -29,6 +29,8 @@ public:
29
29
  DUCKDB_API explicit PendingQueryResult(ErrorData error_message);
30
30
  DUCKDB_API ~PendingQueryResult() override;
31
31
  DUCKDB_API bool AllowStreamResult() const;
32
+ PendingQueryResult(const PendingQueryResult &) = delete;
33
+ PendingQueryResult &operator=(const PendingQueryResult &) = delete;
32
34
 
33
35
  public:
34
36
  //! Executes a single task within the query, returning whether or not the query is ready.
@@ -166,6 +166,13 @@ public:
166
166
  return root.get();
167
167
  }
168
168
 
169
+ //! Provides access to the root of the query tree, but ensures there are no concurrent modifications
170
+ //! This can be useful when implementing continuous profiling or making customizations
171
+ DUCKDB_API void GetRootUnderLock(const std::function<void(optional_ptr<ProfilingNode>)> &callback) {
172
+ lock_guard<std::mutex> guard(lock);
173
+ callback(GetRoot());
174
+ }
175
+
169
176
  private:
170
177
  ClientContext &context;
171
178
 
@@ -103,8 +103,8 @@ private:
103
103
 
104
104
  expression_map_t<unique_ptr<Expression>> stored_expressions;
105
105
  expression_map_t<idx_t> equivalence_set_map;
106
- unordered_map<idx_t, vector<ExpressionValueInformation>> constant_values;
107
- unordered_map<idx_t, vector<reference<Expression>>> equivalence_map;
106
+ map<idx_t, vector<ExpressionValueInformation>> constant_values;
107
+ map<idx_t, vector<reference<Expression>>> equivalence_map;
108
108
  idx_t set_index = 0;
109
109
  //
110
110
  // //! Structures used for OR Filters
@@ -14,6 +14,7 @@
14
14
  namespace duckdb {
15
15
  class LogicalOperator;
16
16
  class LogicalGet;
17
+ class LogicalLimit;
17
18
  class Optimizer;
18
19
 
19
20
  //! Transform
@@ -34,7 +35,7 @@ private:
34
35
  void ReplaceTableReferences(Expression &expr, idx_t new_table_index);
35
36
  unique_ptr<Expression> GetExpression(LogicalOperator &op, idx_t column_index);
36
37
  void ReplaceExpressionReferences(LogicalOperator &next_op, unique_ptr<Expression> &expr);
37
- bool OptimizeLargeLimit(LogicalOperator &child);
38
+ bool OptimizeLargeLimit(LogicalLimit &limit, idx_t limit_val, bool has_offset);
38
39
 
39
40
  private:
40
41
  Optimizer &optimizer;
@@ -29,14 +29,20 @@ struct OptimizerExtensionInput {
29
29
  };
30
30
 
31
31
  typedef void (*optimize_function_t)(OptimizerExtensionInput &input, unique_ptr<LogicalOperator> &plan);
32
+ typedef void (*pre_optimize_function_t)(OptimizerExtensionInput &input, unique_ptr<LogicalOperator> &plan);
32
33
 
33
34
  class OptimizerExtension {
34
35
  public:
35
- //! The parse function of the parser extension.
36
- //! Takes a query string as input and returns ParserExtensionParseData (on success) or an error
37
- optimize_function_t optimize_function;
38
-
39
- //! Additional parser info passed to the parse function
36
+ //! The optimize function of the optimizer extension.
37
+ //! Takes a logical query plan as an input, which it can modify in place
38
+ //! This runs, after the DuckDB optimizers have run
39
+ optimize_function_t optimize_function = nullptr;
40
+ //! The pre-optimize function of the optimizer extension.
41
+ //! Takes a logical query plan as an input, which it can modify in place
42
+ //! This runs, before the DuckDB optimizers have run
43
+ pre_optimize_function_t pre_optimize_function = nullptr;
44
+
45
+ //! Additional optimizer info passed to the optimize functions
40
46
  shared_ptr<OptimizerExtensionInfo> optimizer_info;
41
47
  };
42
48
 
@@ -9,10 +9,10 @@
9
9
  #pragma once
10
10
 
11
11
  #include "duckdb/parallel/task.hpp"
12
- #include "duckdb/parallel/event.hpp"
13
12
  #include "duckdb/common/optional_ptr.hpp"
14
13
 
15
14
  namespace duckdb {
15
+ class Event;
16
16
  class PhysicalOperator;
17
17
  class ThreadContext;
18
18
 
@@ -34,6 +34,9 @@ public:
34
34
  unique_ptr<ThreadContext> thread_context;
35
35
  optional_ptr<const PhysicalOperator> op;
36
36
 
37
+ private:
38
+ ClientContext &context;
39
+
37
40
  public:
38
41
  virtual TaskExecutionResult ExecuteTask(TaskExecutionMode mode) = 0;
39
42
  TaskExecutionResult Execute(TaskExecutionMode mode) override;
@@ -20,7 +20,6 @@
20
20
  namespace duckdb {
21
21
 
22
22
  class Executor;
23
- class Event;
24
23
  class MetaPipeline;
25
24
  class PipelineExecutor;
26
25
  class Pipeline;
@@ -10,6 +10,7 @@
10
10
 
11
11
  #include "duckdb/common/common.hpp"
12
12
  #include "duckdb/common/atomic.hpp"
13
+ #include "duckdb/common/optional_ptr.hpp"
13
14
  #include "duckdb/parallel/task.hpp"
14
15
  #include "duckdb/execution/task_error_manager.hpp"
15
16
 
@@ -47,6 +48,8 @@ private:
47
48
  unique_ptr<ProducerToken> token;
48
49
  atomic<idx_t> completed_tasks;
49
50
  atomic<idx_t> total_tasks;
51
+ friend class BaseExecutorTask;
52
+ optional_ptr<ClientContext> context;
50
53
  };
51
54
 
52
55
  class BaseExecutorTask : public Task {
@@ -0,0 +1,27 @@
1
+ //===----------------------------------------------------------------------===//
2
+ // DuckDB
3
+ //
4
+ // duckdb/parallel/task_notifier.hpp
5
+ //
6
+ //
7
+ //===----------------------------------------------------------------------===//
8
+
9
+ #pragma once
10
+
11
+ #include "duckdb/common/optional_ptr.hpp"
12
+
13
+ namespace duckdb {
14
+ class ClientContext;
15
+
16
+ //! The TaskNotifier notifies ClientContextState listener about started / stopped tasks
17
+ class TaskNotifier {
18
+ public:
19
+ explicit TaskNotifier(optional_ptr<ClientContext> context_p);
20
+
21
+ ~TaskNotifier();
22
+
23
+ private:
24
+ optional_ptr<ClientContext> context;
25
+ };
26
+
27
+ } // namespace duckdb
@@ -68,6 +68,10 @@ public:
68
68
  //! Returns the number of threads
69
69
  DUCKDB_API int32_t NumberOfThreads();
70
70
 
71
+ idx_t GetNumberOfTasks() const;
72
+ idx_t GetProducerCount() const;
73
+ idx_t GetTaskCountForProducer(ProducerToken &token) const;
74
+
71
75
  //! Send signals to n threads, signalling for them to wake up and attempt to execute a task
72
76
  void Signal(idx_t n);
73
77
 
@@ -40,7 +40,7 @@ public:
40
40
  vector<LogicalType> child_types;
41
41
  //! The target LogicalType of the subquery result (i.e. to which type it should be casted, if child_type <>
42
42
  //! child_target). Only used for ANY expressions.
43
- LogicalType child_target;
43
+ vector<LogicalType> child_targets;
44
44
 
45
45
  public:
46
46
  bool HasSubquery() const override {
@@ -9,6 +9,7 @@
9
9
  #pragma once
10
10
 
11
11
  #include "duckdb/planner/bound_tableref.hpp"
12
+ #include "duckdb/common/enums/cte_materialize.hpp"
12
13
 
13
14
  namespace duckdb {
14
15
 
@@ -23,7 +23,7 @@ class TableStatistics;
23
23
  //! Abstraction will support, for example: tiering, versioning, or splitting into multiple block managers.
24
24
  class TableDataWriter {
25
25
  public:
26
- explicit TableDataWriter(TableCatalogEntry &table);
26
+ explicit TableDataWriter(TableCatalogEntry &table, optional_ptr<ClientContext> client_context);
27
27
  virtual ~TableDataWriter();
28
28
 
29
29
  public:
@@ -39,9 +39,11 @@ public:
39
39
 
40
40
  TaskScheduler &GetScheduler();
41
41
  DatabaseInstance &GetDatabase();
42
+ optional_ptr<ClientContext> GetClientContext();
42
43
 
43
44
  protected:
44
45
  DuckTableEntry &table;
46
+ optional_ptr<ClientContext> client_context;
45
47
  //! Pointers to the start of each row group.
46
48
  vector<RowGroupPointer> row_group_pointers;
47
49
  };
@@ -97,7 +97,8 @@ class SingleFileCheckpointWriter final : public CheckpointWriter {
97
97
  friend class SingleFileTableDataWriter;
98
98
 
99
99
  public:
100
- SingleFileCheckpointWriter(AttachedDatabase &db, BlockManager &block_manager, CheckpointType checkpoint_type);
100
+ SingleFileCheckpointWriter(optional_ptr<ClientContext> client_context, AttachedDatabase &db,
101
+ BlockManager &block_manager, CheckpointType checkpoint_type);
101
102
 
102
103
  //! Checkpoint the current state of the WAL and flush it to the main storage. This should be called BEFORE any
103
104
  //! connection is available because right now the checkpointing cannot be done online. (TODO)
@@ -112,10 +113,15 @@ public:
112
113
  return checkpoint_type;
113
114
  }
114
115
 
116
+ optional_ptr<ClientContext> GetClientContext() const {
117
+ return client_context;
118
+ }
119
+
115
120
  public:
116
121
  void WriteTable(TableCatalogEntry &table, Serializer &serializer) override;
117
122
 
118
123
  private:
124
+ optional_ptr<ClientContext> client_context;
119
125
  //! The metadata writer is responsible for writing schema information
120
126
  unique_ptr<MetadataWriter> metadata_writer;
121
127
  //! The table data writer is responsible for writing the DataPointers used by the table chunks
@@ -100,7 +100,8 @@ public:
100
100
  virtual bool AutomaticCheckpoint(idx_t estimated_wal_bytes) = 0;
101
101
  virtual unique_ptr<StorageCommitState> GenStorageCommitState(WriteAheadLog &wal) = 0;
102
102
  virtual bool IsCheckpointClean(MetaBlockPointer checkpoint_id) = 0;
103
- virtual void CreateCheckpoint(CheckpointOptions options = CheckpointOptions()) = 0;
103
+ virtual void CreateCheckpoint(optional_ptr<ClientContext> client_context,
104
+ CheckpointOptions options = CheckpointOptions()) = 0;
104
105
  virtual DatabaseSize GetDatabaseSize() = 0;
105
106
  virtual vector<MetadataBlockInfo> GetMetadataInfo() = 0;
106
107
  virtual shared_ptr<TableIOManager> GetTableIOManager(BoundCreateTableInfo *info) = 0;
@@ -159,7 +160,7 @@ public:
159
160
  bool AutomaticCheckpoint(idx_t estimated_wal_bytes) override;
160
161
  unique_ptr<StorageCommitState> GenStorageCommitState(WriteAheadLog &wal) override;
161
162
  bool IsCheckpointClean(MetaBlockPointer checkpoint_id) override;
162
- void CreateCheckpoint(CheckpointOptions options) override;
163
+ void CreateCheckpoint(optional_ptr<ClientContext> client_context, CheckpointOptions options) override;
163
164
  DatabaseSize GetDatabaseSize() override;
164
165
  vector<MetadataBlockInfo> GetMetadataInfo() override;
165
166
  shared_ptr<TableIOManager> GetTableIOManager(BoundCreateTableInfo *info) override;