duckdb 0.3.5-dev725.0 → 0.3.5-dev750.0

package/src/duckdb.hpp

@@ -11,8 +11,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 #pragma once
 #define DUCKDB_AMALGAMATION 1
 #define DUCKDB_AMALGAMATION_EXTENDED 1
-#define DUCKDB_SOURCE_ID "463e45ba9"
-#define DUCKDB_VERSION "v0.3.5-dev725"
+#define DUCKDB_SOURCE_ID "4e5f39098"
+#define DUCKDB_VERSION "v0.3.5-dev750"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -10479,21 +10479,27 @@ namespace duckdb {
 
 class BaseStatistics;
 class LogicalGet;
-struct ParallelState;
 class TableFilterSet;
 
-struct FunctionOperatorData {
-	DUCKDB_API virtual ~FunctionOperatorData();
-};
-
 struct TableFunctionInfo {
 	DUCKDB_API virtual ~TableFunctionInfo();
 };
 
-struct TableFilterCollection {
-	DUCKDB_API explicit TableFilterCollection(TableFilterSet *table_filters);
+struct GlobalTableFunctionState {
+public:
+	// value returned from MaxThreads when as many threads as possible should be used
+	constexpr static const int64_t MAX_THREADS = 999999999;
+
+public:
+	DUCKDB_API virtual ~GlobalTableFunctionState();
+
+	DUCKDB_API virtual idx_t MaxThreads() const {
+		return 1;
+	}
+};
 
-	TableFilterSet *table_filters;
+struct LocalTableFunctionState {
+	DUCKDB_API virtual ~LocalTableFunctionState();
 };
 
 struct TableFunctionBindInput {
@@ -10511,40 +10517,46 @@ struct TableFunctionBindInput {
 	TableFunctionInfo *info;
 };
 
+struct TableFunctionInitInput {
+	TableFunctionInitInput(const FunctionData *bind_data_p, const vector<column_t> &column_ids_p,
+	                       TableFilterSet *filters_p)
+	    : bind_data(bind_data_p), column_ids(column_ids_p), filters(filters_p) {
+	}
+
+	const FunctionData *bind_data;
+	const vector<column_t> &column_ids;
+	TableFilterSet *filters;
+};
+
+struct TableFunctionInput {
+	TableFunctionInput(const FunctionData *bind_data_p, LocalTableFunctionState *local_state_p,
+	                   GlobalTableFunctionState *global_state_p)
+	    : bind_data(bind_data_p), local_state(local_state_p), global_state(global_state_p) {
+	}
+
+	const FunctionData *bind_data;
+	LocalTableFunctionState *local_state;
+	GlobalTableFunctionState *global_state;
+};
+
 typedef unique_ptr<FunctionData> (*table_function_bind_t)(ClientContext &context, TableFunctionBindInput &input,
                                                           vector<LogicalType> &return_types, vector<string> &names);
-typedef unique_ptr<FunctionOperatorData> (*table_function_init_t)(ClientContext &context, const FunctionData *bind_data,
-                                                                  const vector<column_t> &column_ids,
-                                                                  TableFilterCollection *filters);
+typedef unique_ptr<GlobalTableFunctionState> (*table_function_init_global_t)(ClientContext &context,
+                                                                             TableFunctionInitInput &input);
+typedef unique_ptr<LocalTableFunctionState> (*table_function_init_local_t)(ClientContext &context,
+                                                                           TableFunctionInitInput &input,
+                                                                           GlobalTableFunctionState *global_state);
 typedef unique_ptr<BaseStatistics> (*table_statistics_t)(ClientContext &context, const FunctionData *bind_data,
                                                          column_t column_index);
-typedef void (*table_function_t)(ClientContext &context, const FunctionData *bind_data,
-                                 FunctionOperatorData *operator_state, DataChunk &output);
-
-typedef OperatorResultType (*table_in_out_function_t)(ClientContext &context, const FunctionData *bind_data,
-                                                      FunctionOperatorData *operator_state, DataChunk &input,
-                                                      DataChunk &output);
+typedef void (*table_function_t)(ClientContext &context, TableFunctionInput &data, DataChunk &output);
 
-typedef void (*table_function_parallel_t)(ClientContext &context, const FunctionData *bind_data,
-                                          FunctionOperatorData *operator_state, DataChunk &output,
-                                          ParallelState *parallel_state);
+typedef OperatorResultType (*table_in_out_function_t)(ClientContext &context, TableFunctionInput &data,
+                                                      DataChunk &input, DataChunk &output);
 typedef idx_t (*table_function_get_batch_index_t)(ClientContext &context, const FunctionData *bind_data,
-                                                  FunctionOperatorData *operator_state, ParallelState *parallel_state);
-typedef void (*table_function_cleanup_t)(ClientContext &context, const FunctionData *bind_data,
-                                         FunctionOperatorData *operator_state);
-typedef idx_t (*table_function_max_threads_t)(ClientContext &context, const FunctionData *bind_data);
-typedef unique_ptr<ParallelState> (*table_function_init_parallel_state_t)(ClientContext &context,
-                                                                          const FunctionData *bind_data,
-                                                                          const vector<column_t> &column_ids,
-                                                                          TableFilterCollection *filters);
-typedef unique_ptr<FunctionOperatorData> (*table_function_init_parallel_t)(ClientContext &context,
-                                                                           const FunctionData *bind_data,
-                                                                           ParallelState *state,
-                                                                           const vector<column_t> &column_ids,
-                                                                           TableFilterCollection *filters);
-typedef bool (*table_function_parallel_state_next_t)(ClientContext &context, const FunctionData *bind_data,
-                                                     FunctionOperatorData *state, ParallelState *parallel_state);
-typedef double (*table_function_progress_t)(ClientContext &context, const FunctionData *bind_data);
+                                                  LocalTableFunctionState *local_state,
+                                                  GlobalTableFunctionState *global_state);
+typedef double (*table_function_progress_t)(ClientContext &context, const FunctionData *bind_data,
+                                            const GlobalTableFunctionState *global_state);
 typedef void (*table_function_dependency_t)(unordered_set<CatalogEntry *> &dependencies, const FunctionData *bind_data);
 typedef unique_ptr<NodeStatistics> (*table_function_cardinality_t)(ClientContext &context,
                                                                    const FunctionData *bind_data);
@@ -10557,40 +10569,26 @@ class TableFunction : public SimpleNamedParameterFunction {
 public:
 	DUCKDB_API
 	TableFunction(string name, vector<LogicalType> arguments, table_function_t function,
-	              table_function_bind_t bind = nullptr, table_function_init_t init = nullptr,
-	              table_statistics_t statistics = nullptr, table_function_cleanup_t cleanup = nullptr,
-	              table_function_dependency_t dependency = nullptr, table_function_cardinality_t cardinality = nullptr,
-	              table_function_pushdown_complex_filter_t pushdown_complex_filter = nullptr,
-	              table_function_to_string_t to_string = nullptr, table_function_max_threads_t max_threads = nullptr,
-	              table_function_init_parallel_state_t init_parallel_state = nullptr,
-	              table_function_parallel_t parallel_function = nullptr,
-	              table_function_init_parallel_t parallel_init = nullptr,
-	              table_function_parallel_state_next_t parallel_state_next = nullptr, bool projection_pushdown = false,
-	              bool filter_pushdown = false, table_function_progress_t query_progress = nullptr,
-	              table_in_out_function_t in_out_function = nullptr);
+	              table_function_bind_t bind = nullptr, table_function_init_global_t init_global = nullptr,
+	              table_function_init_local_t init_local = nullptr);
 	DUCKDB_API
 	TableFunction(const vector<LogicalType> &arguments, table_function_t function, table_function_bind_t bind = nullptr,
-	              table_function_init_t init = nullptr, table_statistics_t statistics = nullptr,
-	              table_function_cleanup_t cleanup = nullptr, table_function_dependency_t dependency = nullptr,
-	              table_function_cardinality_t cardinality = nullptr,
-	              table_function_pushdown_complex_filter_t pushdown_complex_filter = nullptr,
-	              table_function_to_string_t to_string = nullptr, table_function_max_threads_t max_threads = nullptr,
-	              table_function_init_parallel_state_t init_parallel_state = nullptr,
-	              table_function_parallel_t parallel_function = nullptr,
-	              table_function_init_parallel_t parallel_init = nullptr,
-	              table_function_parallel_state_next_t parallel_state_next = nullptr, bool projection_pushdown = false,
-	              bool filter_pushdown = false, table_function_progress_t query_progress = nullptr,
-	              table_in_out_function_t in_out_function = nullptr);
+	              table_function_init_global_t init_global = nullptr, table_function_init_local_t init_local = nullptr);
 	DUCKDB_API TableFunction();
 
 	//! Bind function
 	//! This function is used for determining the return type of a table producing function and returning bind data
 	//! The returned FunctionData object should be constant and should not be changed during execution.
 	table_function_bind_t bind;
-	//! (Optional) init function
-	//! Initialize the operator state of the function. The operator state is used to keep track of the progress in the
-	//! table function.
-	table_function_init_t init;
+	//! (Optional) global init function
+	//! Initialize the global operator state of the function.
+	//! The global operator state is used to keep track of the progress in the table function and is shared between
+	//! all threads working on the table function.
+	table_function_init_global_t init_global;
+	//! (Optional) local init function
+	//! Initialize the local operator state of the function.
+	//! The local operator state is used to keep track of the progress in the table function and is thread-local.
+	table_function_init_local_t init_local;
 	//! The main function
 	table_function_t function;
 	//! The table in-out function (if this is an in-out function)
@@ -10598,9 +10596,6 @@ public:
 	//! (Optional) statistics function
 	//! Returns the statistics of a specified column
 	table_statistics_t statistics;
-	//! (Optional) cleanup function
-	//! The final cleanup function, called after all data is exhausted from the main function
-	table_function_cleanup_t cleanup;
 	//! (Optional) dependency function
 	//! Sets up which catalog entries this table function depend on
 	table_function_dependency_t dependency;
@@ -10612,17 +10607,6 @@ public:
 	table_function_pushdown_complex_filter_t pushdown_complex_filter;
 	//! (Optional) function for rendering the operator to a string in profiling output
 	table_function_to_string_t to_string;
-	//! (Optional) function that returns the maximum amount of threads that can work on this task
-	table_function_max_threads_t max_threads;
-	//! (Optional) initialize the parallel scan state, called once in total.
-	table_function_init_parallel_state_t init_parallel_state;
-	//! (Optional) Parallel version of the main function
-	table_function_parallel_t parallel_function;
-	//! (Optional) initialize the parallel scan given the parallel state. Called once per task. Return nullptr if there
-	//! is nothing left to scan.
-	table_function_init_parallel_t parallel_init;
-	//! (Optional) return the next chunk to process in the parallel scan, or return nullptr if there is none
-	table_function_parallel_state_next_t parallel_state_next;
 	//! (Optional) return how much of the table we have scanned up to this point (% of the data)
 	table_function_progress_t table_scan_progress;
 	//! (Optional) returns the current batch index of the current scan operator
@@ -10633,33 +10617,12 @@ public:
 	//! Whether or not the table function supports filter pushdown. If not supported a filter will be added
 	//! that applies the table filter directly.
 	bool filter_pushdown;
-	//! Whether or not the table function supports fetching of a batch index
-	bool supports_batch_index;
 	//! Additional function info, passed to the bind
 	shared_ptr<TableFunctionInfo> function_info;
 };
 
 } // namespace duckdb
 
-//===----------------------------------------------------------------------===//
-//                         DuckDB
-//
-// duckdb/parallel/parallel_state.hpp
-//
-//
-//===----------------------------------------------------------------------===//
-
-
-
-namespace duckdb {
-
-struct ParallelState {
-	virtual ~ParallelState() {
-	}
-};
-
-} // namespace duckdb
-
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -14638,11 +14601,11 @@ typedef unordered_map<block_id_t, unique_ptr<BufferHandle>> buffer_handle_set_t;
 
 struct ColumnScanState {
 	//! The column segment that is currently being scanned
-	ColumnSegment *current;
+	ColumnSegment *current = nullptr;
 	//! The current row index of the scan
-	idx_t row_index;
+	idx_t row_index = 0;
 	//! The internal row index (i.e. the position of the SegmentScanState)
-	idx_t internal_index;
+	idx_t internal_index = 0;
 	//! Segment scan state
 	unique_ptr<SegmentScanState> scan_state;
 	//! Child states of the vector
@@ -14676,10 +14639,10 @@ struct LocalScanState {
 		return storage.get();
 	}
 
-	idx_t chunk_index;
-	idx_t max_index;
-	idx_t last_chunk_count;
-	TableFilterSet *table_filters;
+	idx_t chunk_index = 0;
+	idx_t max_index = 0;
+	idx_t last_chunk_count = 0;
+	TableFilterSet *table_filters = nullptr;
 
 private:
 	shared_ptr<LocalTableStorage> storage;
@@ -14693,17 +14656,13 @@ public:
 	//! The parent scan state
 	TableScanState &parent;
 	//! The current row_group we are scanning
-	RowGroup *row_group;
+	RowGroup *row_group = nullptr;
 	//! The vector index within the row_group
-	idx_t vector_index;
+	idx_t vector_index = 0;
 	//! The maximum row index of this row_group scan
-	idx_t max_row;
+	idx_t max_row = 0;
 	//! Child column scans
 	unique_ptr<ColumnScanState[]> column_scans;
-
-public:
-	//! Move to the next vector, skipping past the current one
-	void NextVector();
 };
 
 class TableScanState {
@@ -14713,7 +14672,7 @@ public:
 	//! The row_group scan state
 	RowGroupScanState row_group_scan_state;
 	//! The total maximum row index
-	idx_t max_row;
+	idx_t max_row = 0;
 	//! The column identifiers of the scan
 	vector<column_t> column_ids;
 	//! The table filters (if any)
@@ -14722,10 +14681,6 @@ public:
 	unique_ptr<AdaptiveFilter> adaptive_filter;
 	//! Transaction-local scan state
 	LocalScanState local_state;
-
-public:
-	//! Move to the next vector
-	void NextVector();
 };
 
 class CreateIndexScanState : public TableScanState {
@@ -16619,6 +16574,15 @@ Sets the init function of the table function
 */
 DUCKDB_API void duckdb_table_function_set_init(duckdb_table_function table_function, duckdb_table_function_init_t init);
 
+/*!
+Sets the thread-local init function of the table function
+
+* table_function: The table function
+* init: The init function
+*/
+DUCKDB_API void duckdb_table_function_set_local_init(duckdb_table_function table_function,
+                                                     duckdb_table_function_init_t init);
+
 /*!
 Sets the main function of the table function
 
@@ -16762,6 +16726,14 @@ This function must be used if projection pushdown is enabled to figure out which
 */
 DUCKDB_API idx_t duckdb_init_get_column_index(duckdb_init_info info, idx_t column_index);
 
+/*!
+Sets how many threads can process this table function in parallel (default: 1)
+
+* info: The info object
+* max_threads: The maximum amount of threads that can process this table function
+*/
+DUCKDB_API void duckdb_init_set_max_threads(duckdb_init_info info, idx_t max_threads);
+
 /*!
 Report that an error has occurred while calling init.
 
@@ -16793,13 +16765,21 @@ For tracking state, use the init data instead.
 DUCKDB_API void *duckdb_function_get_bind_data(duckdb_function_info info);
 
 /*!
-Gets the init data set by `duckdb_bind_set_init_data` during the bind.
+Gets the init data set by `duckdb_init_set_init_data` during the init.
 
 * info: The info object
 * returns: The init data object
 */
 DUCKDB_API void *duckdb_function_get_init_data(duckdb_function_info info);
 
+/*!
+Gets the thread-local init data set by `duckdb_init_set_init_data` during the local_init.
+
+* info: The info object
+* returns: The init data object
+*/
+DUCKDB_API void *duckdb_function_get_local_init_data(duckdb_function_info info);
+
 /*!
 Report that an error has occurred while executing the function.