duckdb 0.8.2-dev11.0 → 0.8.2-dev1212.0

package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp

@@ -74,7 +74,6 @@ public:
 	RowGroupCollection &GetCollection() {
 		return collection.get();
 	}
-	DatabaseInstance &GetDatabase();
 	BlockManager &GetBlockManager();
 	DataTableInfo &GetTableInfo();
 
@@ -82,7 +81,7 @@ public:
 	                               ExpressionExecutor &executor, CollectionScanState &scan_state,
 	                               DataChunk &scan_chunk);
 	unique_ptr<RowGroup> AddColumn(RowGroupCollection &collection, ColumnDefinition &new_column,
-	                               ExpressionExecutor &executor, Expression *default_value, Vector &intermediate);
+	                               ExpressionExecutor &executor, Expression &default_value, Vector &intermediate);
 	unique_ptr<RowGroup> RemoveColumn(RowGroupCollection &collection, idx_t removed_column);
 
 	void CommitDrop();
@@ -123,6 +122,7 @@ public:
 	idx_t Delete(TransactionData transaction, DataTable &table, row_t *row_ids, idx_t count);
 
 	RowGroupWriteData WriteToDisk(PartialBlockManager &manager, const vector<CompressionType> &compression_types);
+	bool AllDeleted();
 	RowGroupPointer Checkpoint(RowGroupWriter &writer, TableStatistics &global_stats);
 	static void Serialize(RowGroupPointer &pointer, Serializer &serializer);
 	static RowGroupPointer Deserialize(Deserializer &source, const vector<LogicalType> &columns);
@@ -170,6 +170,7 @@ struct VersionNode {
 	unique_ptr<ChunkInfo> info[RowGroup::ROW_GROUP_VECTOR_COUNT];
 
 	void SetStart(idx_t start);
+	idx_t GetCommittedDeletedCount(idx_t count);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp

@@ -45,7 +45,6 @@ public:
 	void AppendRowGroup(SegmentLock &l, idx_t start_row);
 	//! Get the nth row-group, negative numbers start from the back (so -1 is the last row group, etc)
 	RowGroup *GetRowGroup(int64_t index);
-	idx_t RowGroupCount();
 	void Verify();
 
 	void InitializeScan(CollectionScanState &state, const vector<column_t> &column_ids, TableFilterSet *table_filters);
@@ -93,7 +92,7 @@ public:
 	const vector<LogicalType> &GetTypes() const;
 
 	shared_ptr<RowGroupCollection> AddColumn(ClientContext &context, ColumnDefinition &new_column,
-	                                         Expression *default_value);
+	                                         Expression &default_value);
 	shared_ptr<RowGroupCollection> RemoveColumn(idx_t col_idx);
 	shared_ptr<RowGroupCollection> AlterType(ClientContext &context, idx_t changed_idx, const LogicalType &target_type,
 	                                         vector<column_t> bound_columns, Expression &cast_expr);
@@ -104,7 +103,6 @@ public:
 	void SetDistinct(column_t column_id, unique_ptr<DistinctStatistics> distinct_stats);
 
 	AttachedDatabase &GetAttached();
-	DatabaseInstance &GetDatabase();
 	BlockManager &GetBlockManager() {
 		return block_manager;
 	}

package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp

@@ -32,7 +32,7 @@ public:
 	LocalTableStorage(DataTable &table, LocalTableStorage &parent, idx_t drop_idx);
 	// Create a LocalTableStorage from an ADD COLUMN
 	LocalTableStorage(ClientContext &context, DataTable &table, LocalTableStorage &parent, ColumnDefinition &new_column,
-	                  optional_ptr<Expression> default_value);
+	                  Expression &default_value);
 	~LocalTableStorage();
 
 	reference<DataTable> table_ref;
@@ -143,8 +143,7 @@ public:
 
 	idx_t AddedRows(DataTable &table);
 
-	void AddColumn(DataTable &old_dt, DataTable &new_dt, ColumnDefinition &new_column,
-	               optional_ptr<Expression> default_value);
+	void AddColumn(DataTable &old_dt, DataTable &new_dt, ColumnDefinition &new_column, Expression &default_value);
 	void DropColumn(DataTable &old_dt, DataTable &new_dt, idx_t removed_column);
 	void ChangeType(DataTable &old_dt, DataTable &new_dt, idx_t changed_idx, const LogicalType &target_type,
 	                const vector<column_t> &bound_columns, Expression &cast_expr);

package/src/duckdb/src/include/duckdb.h

@@ -279,6 +279,9 @@ typedef struct _duckdb_appender {
 typedef struct _duckdb_arrow {
 	void *__arrw;
 } * duckdb_arrow;
+typedef struct _duckdb_arrow_stream {
+	void *__arrwstr;
+} * duckdb_arrow_stream;
 typedef struct _duckdb_config {
 	void *__cnfg;
 } * duckdb_config;
@@ -1078,6 +1081,31 @@ Executes the prepared statement with the given bound parameters, and returns an
 DUCKDB_API duckdb_state duckdb_execute_prepared_arrow(duckdb_prepared_statement prepared_statement,
                                                       duckdb_arrow *out_result);
 
+/*!
+Scans the Arrow stream and creates a view with the given name.
+
+* connection: The connection on which to execute the scan.
+* table_name: Name of the temporary view to create.
+* arrow: Arrow stream wrapper.
+* returns: `DuckDBSuccess` on success or `DuckDBError` on failure.
+*/
+DUCKDB_API duckdb_state duckdb_arrow_scan(duckdb_connection connection, const char *table_name,
+                                          duckdb_arrow_stream arrow);
+
+/*!
+Scans the Arrow array and creates a view with the given name.
+
+* connection: The connection on which to execute the scan.
+* table_name: Name of the temporary view to create.
+* arrow_schema: Arrow schema wrapper.
+* arrow_array: Arrow array wrapper.
+* out_stream: Output array stream that wraps around the passed schema, for releasing/deleting once done.
+* returns: `DuckDBSuccess` on success or `DuckDBError` on failure.
+*/
+DUCKDB_API duckdb_state duckdb_arrow_array_scan(duckdb_connection connection, const char *table_name,
+                                                duckdb_arrow_schema arrow_schema, duckdb_arrow_array arrow_array,
+                                                duckdb_arrow_stream *out_stream);
+
 //===--------------------------------------------------------------------===//
 // Extract Statements
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/main/capi/arrow-c.cpp

@@ -1,5 +1,6 @@
-#include "duckdb/main/capi/capi_internal.hpp"
 #include "duckdb/common/arrow/arrow_converter.hpp"
+#include "duckdb/function/table/arrow.hpp"
+#include "duckdb/main/capi/capi_internal.hpp"
 
 using duckdb::ArrowConverter;
 using duckdb::ArrowResultWrapper;
@@ -108,3 +109,156 @@ duckdb_state duckdb_execute_prepared_arrow(duckdb_prepared_statement prepared_st
 	*out_result = reinterpret_cast<duckdb_arrow>(arrow_wrapper);
 	return !arrow_wrapper->result->HasError() ? DuckDBSuccess : DuckDBError;
 }
+
+namespace arrow_array_stream_wrapper {
+namespace {
+struct PrivateData {
+	ArrowSchema *schema;
+	ArrowArray *array;
+	bool done = false;
+};
+
+// LCOV_EXCL_START
+// This function is never called, but used to set ArrowSchema's release functions to a non-null NOOP.
+void EmptySchemaRelease(ArrowSchema *) {
+}
+// LCOV_EXCL_STOP
+
+void EmptyArrayRelease(ArrowArray *) {
+}
+
+void EmptyStreamRelease(ArrowArrayStream *) {
+}
+
+void FactoryGetSchema(uintptr_t stream_factory_ptr, duckdb::ArrowSchemaWrapper &schema) {
+	auto private_data =
+	    reinterpret_cast<PrivateData *>(reinterpret_cast<ArrowArrayStream *>(stream_factory_ptr)->private_data);
+	schema.arrow_schema = *private_data->schema;
+
+	// Need to nullify the root schema's release function here, because streams don't allow us to set the release
+	// function. For the schema's children, we nullify the release functions in `duckdb_arrow_scan`, so we don't need to
+	// handle them again here. We set this to nullptr and not EmptySchemaRelease to prevent ArrowSchemaWrapper's
+	// destructor from destroying the schema (it's the caller's responsibility).
+	schema.arrow_schema.release = nullptr;
+}
+
+int GetSchema(struct ArrowArrayStream *stream, struct ArrowSchema *out) {
+	auto private_data = static_cast<arrow_array_stream_wrapper::PrivateData *>((stream->private_data));
+	if (private_data->schema == nullptr) {
+		return DuckDBError;
+	}
+
+	*out = *private_data->schema;
+	out->release = EmptySchemaRelease;
+	return DuckDBSuccess;
+}
+
+int GetNext(struct ArrowArrayStream *stream, struct ArrowArray *out) {
+	auto private_data = static_cast<arrow_array_stream_wrapper::PrivateData *>((stream->private_data));
+	*out = *private_data->array;
+	if (private_data->done) {
+		out->release = nullptr;
+	} else {
+		out->release = EmptyArrayRelease;
+	}
+
+	private_data->done = true;
+	return DuckDBSuccess;
+}
+
+duckdb::unique_ptr<duckdb::ArrowArrayStreamWrapper> FactoryGetNext(uintptr_t stream_factory_ptr,
+                                                                   duckdb::ArrowStreamParameters &parameters) {
+	auto stream = reinterpret_cast<ArrowArrayStream *>(stream_factory_ptr);
+	auto ret = duckdb::make_uniq<duckdb::ArrowArrayStreamWrapper>();
+	ret->arrow_array_stream = *stream;
+	ret->arrow_array_stream.release = EmptyStreamRelease;
+	return ret;
+}
+
+// LCOV_EXCL_START
+// This function is never be called, because it's used to construct a stream wrapping around a caller-supplied
+// ArrowArray. Thus, the stream itself cannot produce an error.
+const char *GetLastError(struct ArrowArrayStream *stream) {
+	return nullptr;
+}
+// LCOV_EXCL_STOP
+
+void Release(struct ArrowArrayStream *stream) {
+	if (stream->private_data != nullptr) {
+		delete reinterpret_cast<PrivateData *>(stream->private_data);
+	}
+
+	stream->private_data = nullptr;
+	stream->release = nullptr;
+}
+
+duckdb_state Ingest(duckdb_connection connection, const char *table_name, struct ArrowArrayStream *input) {
+	try {
+		auto cconn = reinterpret_cast<duckdb::Connection *>(connection);
+		cconn
+		    ->TableFunction("arrow_scan", {duckdb::Value::POINTER((uintptr_t)input),
+		                                   duckdb::Value::POINTER((uintptr_t)FactoryGetNext),
+		                                   duckdb::Value::POINTER((uintptr_t)FactoryGetSchema)})
+		    ->CreateView(table_name, true, false);
+	} catch (...) { // LCOV_EXCL_START
+		// Tried covering this in tests, but it proved harder than expected. At the time of writing:
+		// - Passing any name to `CreateView` worked without throwing an exception
+		// - Passing a null Arrow array worked without throwing an exception
+		// - Passing an invalid schema (without any columns) led to an InternalException with SIGABRT, which is meant to
+		//   be un-catchable. This case likely needs to be handled gracefully within `arrow_scan`.
+		// Ref: https://discord.com/channels/909674491309850675/921100573732909107/1115230468699336785
+		return DuckDBError;
+	} // LCOV_EXCL_STOP
+
+	return DuckDBSuccess;
+}
+} // namespace
+} // namespace arrow_array_stream_wrapper
+
+duckdb_state duckdb_arrow_scan(duckdb_connection connection, const char *table_name, duckdb_arrow_stream arrow) {
+	auto stream = reinterpret_cast<ArrowArrayStream *>(arrow);
+
+	// Backup release functions - we nullify children schema release functions because we don't want to release on
+	// behalf of the caller, downstream in our code. Note that Arrow releases target immediate children, but aren't
+	// recursive. So we only back up immediate children here and restore their functions.
+	ArrowSchema schema;
+	if (stream->get_schema(stream, &schema) == DuckDBError) {
+		return DuckDBError;
+	}
+
+	typedef void (*release_fn_t)(ArrowSchema *);
+	std::vector<release_fn_t> release_fns(schema.n_children);
+	for (int64_t i = 0; i < schema.n_children; i++) {
+		auto child = schema.children[i];
+		release_fns[i] = child->release;
+		child->release = arrow_array_stream_wrapper::EmptySchemaRelease;
+	}
+
+	auto ret = arrow_array_stream_wrapper::Ingest(connection, table_name, stream);
+
+	// Restore release functions.
+	for (int64_t i = 0; i < schema.n_children; i++) {
+		schema.children[i]->release = release_fns[i];
+	}
+
+	return ret;
+}
+
+duckdb_state duckdb_arrow_array_scan(duckdb_connection connection, const char *table_name,
+                                     duckdb_arrow_schema arrow_schema, duckdb_arrow_array arrow_array,
+                                     duckdb_arrow_stream *out_stream) {
+	auto private_data = new arrow_array_stream_wrapper::PrivateData;
+	private_data->schema = reinterpret_cast<ArrowSchema *>(arrow_schema);
+	private_data->array = reinterpret_cast<ArrowArray *>(arrow_array);
+	private_data->done = false;
+
+	ArrowArrayStream *stream = new ArrowArrayStream;
+	*out_stream = reinterpret_cast<duckdb_arrow_stream>(stream);
+	stream->get_schema = arrow_array_stream_wrapper::GetSchema;
+	stream->get_next = arrow_array_stream_wrapper::GetNext;
+	stream->get_last_error = arrow_array_stream_wrapper::GetLastError;
+	stream->release = arrow_array_stream_wrapper::Release;
+	stream->private_data = private_data;
+
+	return duckdb_arrow_scan(connection, table_name, reinterpret_cast<duckdb_arrow_stream>(stream));
+}

package/src/duckdb/src/main/capi/duckdb_value-c.cpp

@@ -26,7 +26,7 @@ char *duckdb_get_varchar(duckdb_value value) {
 	auto str_val = val->DefaultCastAs(duckdb::LogicalType::VARCHAR);
 	auto &str = duckdb::StringValue::Get(str_val);
 
-	auto result = reinterpret_cast<char *>(malloc(sizeof(char *) * (str.size() + 1)));
+	auto result = reinterpret_cast<char *>(malloc(sizeof(char) * (str.size() + 1)));
 	memcpy(result, str.c_str(), str.size());
 	result[str.size()] = '\0';
 	return result;

package/src/duckdb/src/main/config.cpp

@@ -92,6 +92,7 @@ static ConfigurationOption internal_options[] = {DUCKDB_GLOBAL(AccessModeSetting
                                                  DUCKDB_LOCAL(OrderedAggregateThreshold),
                                                  DUCKDB_GLOBAL(PasswordSetting),
                                                  DUCKDB_LOCAL(PerfectHashThresholdSetting),
+                                                 DUCKDB_LOCAL(PivotFilterThreshold),
                                                  DUCKDB_LOCAL(PivotLimitSetting),
                                                  DUCKDB_LOCAL(PreserveIdentifierCase),
                                                  DUCKDB_GLOBAL(PreserveInsertionOrder),
@@ -109,6 +110,7 @@ static ConfigurationOption internal_options[] = {DUCKDB_GLOBAL(AccessModeSetting
                                                  DUCKDB_GLOBAL_ALIAS("user", UsernameSetting),
                                                  DUCKDB_GLOBAL_ALIAS("wal_autocheckpoint", CheckpointThresholdSetting),
                                                  DUCKDB_GLOBAL_ALIAS("worker_threads", ThreadsSetting),
+                                                 DUCKDB_GLOBAL(FlushAllocatorSetting),
                                                  FINAL_SETTING};
 
 vector<ConfigurationOption> DBConfig::GetOptions() {

package/src/duckdb/src/main/database.cpp

@@ -178,7 +178,7 @@ void DatabaseInstance::CreateMainDatabase() {
 void ThrowExtensionSetUnrecognizedOptions(const unordered_map<string, Value> &unrecognized_options) {
 	auto unrecognized_options_iter = unrecognized_options.begin();
 	string unrecognized_option_keys = unrecognized_options_iter->first;
-	for (; unrecognized_options_iter == unrecognized_options.end(); ++unrecognized_options_iter) {
+	while (++unrecognized_options_iter != unrecognized_options.end()) {
 		unrecognized_option_keys = "," + unrecognized_options_iter->first;
 	}
 	throw InvalidInputException("Unrecognized configuration property \"%s\"", unrecognized_option_keys);

package/src/duckdb/src/main/extension/extension_helper.cpp

@@ -6,88 +6,85 @@
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/main/database.hpp"
 
-#if defined(BUILD_ICU_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
-#define ICU_STATICALLY_LOADED true
-#include "icu_extension.hpp"
-#else
-#define ICU_STATICALLY_LOADED false
+// Note that c++ preprocessor doesn't have a nice way to clean this up so we need to set the defines we use to false
+// explicitly when they are undefined
+#ifndef DUCKDB_EXTENSION_ICU_LINKED
+#define DUCKDB_EXTENSION_ICU_LINKED false
 #endif
 
-#if defined(BUILD_PARQUET_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
-#define PARQUET_STATICALLY_LOADED true
-#include "parquet_extension.hpp"
-#else
-#define PARQUET_STATICALLY_LOADED false
+#ifndef DUCKDB_EXTENSION_PARQUET_LINKED
+#define DUCKDB_EXTENSION_PARQUET_LINKED false
 #endif
 
-#if defined(BUILD_TPCH_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
-#define TPCH_STATICALLY_LOADED true
-#include "tpch_extension.hpp"
-#else
-#define TPCH_STATICALLY_LOADED false
+#ifndef DUCKDB_EXTENSION_TPCH_LINKED
+#define DUCKDB_EXTENSION_TPCH_LINKED false
 #endif
 
-#if defined(BUILD_TPCDS_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
-#define TPCDS_STATICALLY_LOADED true
-#include "tpcds_extension.hpp"
-#else
-#define TPCDS_STATICALLY_LOADED false
+#ifndef DUCKDB_EXTENSION_TPCDS_LINKED
+#define DUCKDB_EXTENSION_TPCDS_LINKED false
 #endif
 
-#if defined(BUILD_FTS_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
-#define FTS_STATICALLY_LOADED true
-#include "fts_extension.hpp"
-#else
-#define FTS_STATICALLY_LOADED false
+#ifndef DUCKDB_EXTENSION_FTS_LINKED
+#define DUCKDB_EXTENSION_FTS_LINKED false
 #endif
 
-#if defined(BUILD_HTTPFS_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
-#define HTTPFS_STATICALLY_LOADED true
-#include "httpfs_extension.hpp"
-#else
-#define HTTPFS_STATICALLY_LOADED false
+#ifndef DUCKDB_EXTENSION_HTTPFS_LINKED
+#define DUCKDB_EXTENSION_HTTPFS_LINKED false
 #endif
 
-#if defined(BUILD_VISUALIZER_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
-#include "visualizer_extension.hpp"
+#ifndef DUCKDB_EXTENSION_JSON_LINKED
+#define DUCKDB_EXTENSION_JSON_LINKED false
 #endif
 
-#if defined(BUILD_JSON_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
-#define JSON_STATICALLY_LOADED true
-#include "json_extension.hpp"
-#else
-#define JSON_STATICALLY_LOADED false
+#ifndef DUCKDB_EXTENSION_JEMALLOC_LINKED
+#define DUCKDB_EXTENSION_JEMALLOC_LINKED false
 #endif
 
-#if defined(BUILD_JEMALLOC_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
-#define JEMALLOC_STATICALLY_LOADED true
-#include "jemalloc_extension.hpp"
+#ifndef DUCKDB_EXTENSION_AUTOCOMPLETE_LINKED
+#define DUCKDB_EXTENSION_AUTOCOMPLETE_LINKED false
+#endif
+
+// Load the generated header file containing our list of extension headers
+#if defined(GENERATED_EXTENSION_HEADERS) && GENERATED_EXTENSION_HEADERS && !defined(DUCKDB_AMALGAMATION)
+#include "generated_extension_loader.hpp"
 #else
-#define JEMALLOC_STATICALLY_LOADED false
+// TODO: rewrite package_build.py to allow also loading out-of-tree extensions in non-cmake builds, after that
+//		 these can be removed
+#if DUCKDB_EXTENSION_ICU_LINKED
+#include "icu_extension.hpp"
 #endif
 
-#if defined(BUILD_EXCEL_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
-#include "excel_extension.hpp"
+#if DUCKDB_EXTENSION_PARQUET_LINKED
+#include "parquet_extension.hpp"
 #endif
 
-#if defined(BUILD_SQLSMITH_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
-#include "sqlsmith_extension.hpp"
+#if DUCKDB_EXTENSION_TPCH_LINKED
+#include "tpch_extension.hpp"
 #endif
 
-#if defined(BUILD_INET_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
-#include "inet_extension.hpp"
+#if DUCKDB_EXTENSION_TPCDS_LINKED
+#include "tpcds_extension.hpp"
 #endif
 
-#if defined(BUILD_AUTOCOMPLETE_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
-#define AUTOCOMPLETE_STATICALLY_LOADED true
-#include "autocomplete_extension.hpp"
-#else
-#define AUTOCOMPLETE_STATICALLY_LOADED false
+#if DUCKDB_EXTENSION_FTS_LINKED
+#include "fts_extension.hpp"
 #endif
 
-// Load the generated header file containing our list of extension headers
-#if defined(OOTE_HEADERS_AVAILABLE) && OOTE_HEADERS_AVAILABLE
-#include "extension_oote_loader.hpp"
+#if DUCKDB_EXTENSION_HTTPFS_LINKED
+#include "httpfs_extension.hpp"
+#endif
+
+#if DUCKDB_EXTENSION_JSON_LINKED
+#include "json_extension.hpp"
+#endif
+
+#if DUCKDB_EXTENSION_JEMALLOC_LINKED
+#include "jemalloc_extension.hpp"
+#endif
+
+#if DUCKDB_EXTENSION_AUTOCOMPLETE_LINKED
+#include "autocomplete_extension.hpp"
+#endif
 #endif
 
 namespace duckdb {
@@ -96,15 +93,15 @@ namespace duckdb {
 // Default Extensions
 //===--------------------------------------------------------------------===//
 static DefaultExtension internal_extensions[] = {
-    {"icu", "Adds support for time zones and collations using the ICU library", ICU_STATICALLY_LOADED},
-    {"parquet", "Adds support for reading and writing parquet files", PARQUET_STATICALLY_LOADED},
-    {"tpch", "Adds TPC-H data generation and query support", TPCH_STATICALLY_LOADED},
-    {"tpcds", "Adds TPC-DS data generation and query support", TPCDS_STATICALLY_LOADED},
-    {"fts", "Adds support for Full-Text Search Indexes", FTS_STATICALLY_LOADED},
-    {"httpfs", "Adds support for reading and writing files over a HTTP(S) connection", HTTPFS_STATICALLY_LOADED},
-    {"json", "Adds support for JSON operations", JSON_STATICALLY_LOADED},
-    {"jemalloc", "Overwrites system allocator with JEMalloc", JEMALLOC_STATICALLY_LOADED},
-    {"autocomplete", "Add supports for autocomplete in the shell", AUTOCOMPLETE_STATICALLY_LOADED},
+    {"icu", "Adds support for time zones and collations using the ICU library", DUCKDB_EXTENSION_ICU_LINKED},
+    {"parquet", "Adds support for reading and writing parquet files", DUCKDB_EXTENSION_PARQUET_LINKED},
+    {"tpch", "Adds TPC-H data generation and query support", DUCKDB_EXTENSION_TPCH_LINKED},
+    {"tpcds", "Adds TPC-DS data generation and query support", DUCKDB_EXTENSION_TPCDS_LINKED},
+    {"fts", "Adds support for Full-Text Search Indexes", DUCKDB_EXTENSION_FTS_LINKED},
+    {"httpfs", "Adds support for reading and writing files over a HTTP(S) connection", DUCKDB_EXTENSION_HTTPFS_LINKED},
+    {"json", "Adds support for JSON operations", DUCKDB_EXTENSION_JSON_LINKED},
+    {"jemalloc", "Overwrites system allocator with JEMalloc", DUCKDB_EXTENSION_JEMALLOC_LINKED},
+    {"autocomplete", "Add supports for autocomplete in the shell", DUCKDB_EXTENSION_AUTOCOMPLETE_LINKED},
     {"motherduck", "Enables motherduck integration with the system", false},
     {"sqlite_scanner", "Adds support for reading SQLite database files", false},
     {"postgres_scanner", "Adds support for reading from a Postgres database", false},
@@ -143,15 +140,20 @@ bool ExtensionHelper::AllowAutoInstall(const string &extension) {
 // Load Statically Compiled Extension
 //===--------------------------------------------------------------------===//
 void ExtensionHelper::LoadAllExtensions(DuckDB &db) {
+	// The in-tree extensions that we check. Non-cmake builds are currently limited to these for static linking
+	// TODO: rewrite package_build.py to allow also loading out-of-tree extensions in non-cmake builds, after that
+	//		 these can be removed
 	unordered_set<string> extensions {"parquet", "icu",   "tpch",     "tpcds", "fts",      "httpfs",      "visualizer",
 	                                  "json",    "excel", "sqlsmith", "inet",  "jemalloc", "autocomplete"};
 	for (auto &ext : extensions) {
 		LoadExtensionInternal(db, ext, true);
 	}
 
-#if defined(OOTE_HEADERS_AVAILABLE) && OOTE_HEADERS_AVAILABLE
-	for (auto &ext : OOT_EXTENSIONS) {
-		LoadExtensionInternal(db, ext, true);
+#if defined(GENERATED_EXTENSION_HEADERS) && GENERATED_EXTENSION_HEADERS
+	for (auto &ext : LINKED_EXTENSIONS) {
+		if (extensions.find(ext) != extensions.end()) {
+			LoadExtensionInternal(db, ext, true);
+		}
 	}
 #endif
 }
@@ -178,105 +180,110 @@ ExtensionLoadResult ExtensionHelper::LoadExtensionInternal(DuckDB &db, const std
 		return ExtensionLoadResult::LOADED_EXTENSION;
 	}
 #endif
+
+	// This is the main extension loading mechanism that loads the extension that are statically linked.
+#if defined(GENERATED_EXTENSION_HEADERS) && GENERATED_EXTENSION_HEADERS
+	if (TryLoadLinkedExtension(db, extension)) {
+		return ExtensionLoadResult::LOADED_EXTENSION;
+	} else {
+		return ExtensionLoadResult::NOT_LOADED;
+	}
+#endif
+
+	// This is the fallback to the "old" extension loading mechanism for non-cmake builds
+	// TODO: rewrite package_build.py to allow also loading out-of-tree extensions in non-cmake builds
 	if (extension == "parquet") {
-#if PARQUET_STATICALLY_LOADED
+#if DUCKDB_EXTENSION_PARQUET_LINKED
 		db.LoadExtension<ParquetExtension>();
 #else
 		// parquet extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "icu") {
-#if ICU_STATICALLY_LOADED
+#if DUCKDB_EXTENSION_ICU_LINKED
 		db.LoadExtension<IcuExtension>();
 #else
 		// icu extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "tpch") {
-#if TPCH_STATICALLY_LOADED
+#if DUCKDB_EXTENSION_TPCH_LINKED
 		db.LoadExtension<TpchExtension>();
 #else
 		// icu extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "tpcds") {
-#if TPCDS_STATICALLY_LOADED
+#if DUCKDB_EXTENSION_TPCDS_LINKED
 		db.LoadExtension<TpcdsExtension>();
 #else
 		// icu extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "fts") {
-#if FTS_STATICALLY_LOADED
-		db.LoadExtension<FtsExtension>();
+#if DUCKDB_EXTENSION_FTS_LINKED
+//		db.LoadExtension<FtsExtension>();
 #else
 		// fts extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "httpfs") {
-#if HTTPFS_STATICALLY_LOADED
+#if DUCKDB_EXTENSION_HTTPFS_LINKED
 		db.LoadExtension<HttpfsExtension>();
 #else
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "visualizer") {
-#if defined(BUILD_VISUALIZER_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
+#if DUCKDB_EXTENSION_VISUALIZER_LINKED
 		db.LoadExtension<VisualizerExtension>();
 #else
 		// visualizer extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "json") {
-#if JSON_STATICALLY_LOADED
+#if DUCKDB_EXTENSION_JSON_LINKED
 		db.LoadExtension<JsonExtension>();
 #else
 		// json extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "excel") {
-#if defined(BUILD_EXCEL_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
+#if DUCKDB_EXTENSION_EXCEL_LINKED
 		db.LoadExtension<ExcelExtension>();
 #else
 		// excel extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "sqlsmith") {
-#if defined(BUILD_SQLSMITH_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
+#if DUCKDB_EXTENSION_SQLSMITH_LINKED
 		db.LoadExtension<SqlsmithExtension>();
 #else
 		// excel extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "jemalloc") {
-#if defined(BUILD_JEMALLOC_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
+#if DUCKDB_EXTENSION_JEMALLOC_LINKED
 		db.LoadExtension<JemallocExtension>();
 #else
 		// jemalloc extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "autocomplete") {
-#if defined(BUILD_AUTOCOMPLETE_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
+#if DUCKDB_EXTENSION_AUTOCOMPLETE_LINKED
 		db.LoadExtension<AutocompleteExtension>();
 #else
 		// autocomplete extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
 	} else if (extension == "inet") {
-#if defined(BUILD_INET_EXTENSION) && !defined(DISABLE_BUILTIN_EXTENSIONS)
+#if DUCKDB_EXTENSION_INET_LINKED
 		db.LoadExtension<InetExtension>();
 #else
 		// inet extension required but not build: skip this test
 		return ExtensionLoadResult::NOT_LOADED;
 #endif
-	} else {
-
-#if defined(OOTE_HEADERS_AVAILABLE) && OOTE_HEADERS_AVAILABLE
-		if (TryLoadLinkedExtension(db, extension)) {
-			return ExtensionLoadResult::LOADED_EXTENSION;
-		}
-#endif
-		return ExtensionLoadResult::EXTENSION_UNKNOWN;
 	}
+
 	return ExtensionLoadResult::LOADED_EXTENSION;
 }