duckdb 0.8.1-dev416.0 → 0.8.1-dev477.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.8.1-dev416.0",
+  "version": "0.8.1-dev477.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {
@@ -28,11 +28,13 @@
   },
   "devDependencies": {
     "@types/chai": "^4.3.4",
+    "@types/chai-as-promised": "^7.1.5",
     "@types/mocha": "^10.0.0",
     "@types/node": "^18.11.0",
     "apache-arrow": "^9.0.0",
     "aws-sdk": "^2.790.0",
     "chai": "^4.3.6",
+    "chai-as-promised": "^7.1.1",
     "jsdoc3-parser": "^2.0.0",
     "mocha": "^8.3.0",
     "ts-node": "^10.9.1",

package/src/duckdb/src/common/arrow/arrow_converter.cpp

@@ -16,7 +16,7 @@
 namespace duckdb {
 
 void ArrowConverter::ToArrowArray(DataChunk &input, ArrowArray *out_array, ArrowOptions options) {
-	ArrowAppender appender(input.GetTypes(), input.size(), options);
+	ArrowAppender appender(input.GetTypes(), input.size(), std::move(options));
 	appender.Append(input, 0, input.size(), input.size());
 	*out_array = appender.Finalize();
 }
@@ -59,10 +59,10 @@ void InitializeChild(ArrowSchema &child, const string &name = "") {
 	child.dictionary = nullptr;
 }
 void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, const LogicalType &type,
-                    const string &config_timezone, ArrowOptions options);
+                    const ArrowOptions &options);
 
 void SetArrowMapFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, const LogicalType &type,
-                       const string &config_timezone, ArrowOptions options) {
+                       const ArrowOptions &options) {
 	child.format = "+m";
 	//! Map has one child which is a struct
 	child.n_children = 1;
@@ -73,11 +73,11 @@ void SetArrowMapFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child,
 	InitializeChild(root_holder.nested_children.back()[0]);
 	child.children = &root_holder.nested_children_ptr.back()[0];
 	child.children[0]->name = "entries";
-	SetArrowFormat(root_holder, **child.children, ListType::GetChildType(type), config_timezone, options);
+	SetArrowFormat(root_holder, **child.children, ListType::GetChildType(type), options);
 }
 
 void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, const LogicalType &type,
-                    const string &config_timezone, ArrowOptions options) {
+                    const ArrowOptions &options) {
 	switch (type.id()) {
 	case LogicalTypeId::BOOLEAN:
 		child.format = "b";
@@ -134,7 +134,7 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
 		child.format = "tsu:";
 		break;
 	case LogicalTypeId::TIMESTAMP_TZ: {
-		string format = "tsu:" + config_timezone;
+		string format = "tsu:" + options.time_zone;
 		auto format_ptr = make_unsafe_uniq_array<char>(format.size() + 1);
 		for (size_t i = 0; i < format.size(); i++) {
 			format_ptr[i] = format[i];
@@ -192,7 +192,7 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
 		InitializeChild(root_holder.nested_children.back()[0]);
 		child.children = &root_holder.nested_children_ptr.back()[0];
 		child.children[0]->name = "l";
-		SetArrowFormat(root_holder, **child.children, ListType::GetChildType(type), config_timezone, options);
+		SetArrowFormat(root_holder, **child.children, ListType::GetChildType(type), options);
 		break;
 	}
 	case LogicalTypeId::STRUCT: {
@@ -220,13 +220,12 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
 			root_holder.owned_type_names.push_back(std::move(name_ptr));
 
 			child.children[type_idx]->name = root_holder.owned_type_names.back().get();
-			SetArrowFormat(root_holder, *child.children[type_idx], child_types[type_idx].second, config_timezone,
-			               options);
+			SetArrowFormat(root_holder, *child.children[type_idx], child_types[type_idx].second, options);
 		}
 		break;
 	}
 	case LogicalTypeId::MAP: {
-		SetArrowMapFormat(root_holder, child, type, config_timezone, options);
+		SetArrowMapFormat(root_holder, child, type, options);
 		break;
 	}
 	case LogicalTypeId::ENUM: {
@@ -259,7 +258,7 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
 }
 
 void ArrowConverter::ToArrowSchema(ArrowSchema *out_schema, const vector<LogicalType> &types,
-                                   const vector<string> &names, const string &config_timezone, ArrowOptions options) {
+                                   const vector<string> &names, const ArrowOptions &options) {
 	D_ASSERT(out_schema);
 	D_ASSERT(types.size() == names.size());
 	idx_t column_count = types.size();
@@ -287,7 +286,7 @@ void ArrowConverter::ToArrowSchema(ArrowSchema *out_schema, const vector<Logical
 
 		auto &child = root_holder->children[col_idx];
 		InitializeChild(child, names[col_idx]);
-		SetArrowFormat(*root_holder, child, types[col_idx], config_timezone, options);
+		SetArrowFormat(*root_holder, child, types[col_idx], options);
 	}
 
 	// Release ownership to caller

package/src/duckdb/src/common/arrow/arrow_wrapper.cpp

@@ -80,7 +80,7 @@ int ResultArrowArrayStreamWrapper::MyStreamGetSchema(struct ArrowArrayStream *st
 	auto my_stream = reinterpret_cast<ResultArrowArrayStreamWrapper *>(stream->private_data);
 	if (!my_stream->column_types.empty()) {
 		ArrowConverter::ToArrowSchema(out, my_stream->column_types, my_stream->column_names,
-		                              my_stream->timezone_config);
+		                              QueryResult::GetArrowOptions(*my_stream->result));
 		return 0;
 	}
 
@@ -100,7 +100,8 @@ int ResultArrowArrayStreamWrapper::MyStreamGetSchema(struct ArrowArrayStream *st
 		my_stream->column_types = result.types;
 		my_stream->column_names = result.names;
 	}
-	ArrowConverter::ToArrowSchema(out, my_stream->column_types, my_stream->column_names, my_stream->timezone_config);
+	ArrowConverter::ToArrowSchema(out, my_stream->column_types, my_stream->column_names,
+	                              QueryResult::GetArrowOptions(*my_stream->result));
 	return 0;
 }
 
@@ -186,7 +187,7 @@ bool ArrowUtil::TryFetchNext(QueryResult &result, unique_ptr<DataChunk> &chunk,
 bool ArrowUtil::TryFetchChunk(QueryResult *result, idx_t chunk_size, ArrowArray *out, idx_t &count,
                               PreservedError &error) {
 	count = 0;
-	ArrowAppender appender(result->types, chunk_size);
+	ArrowAppender appender(result->types, chunk_size, QueryResult::GetArrowOptions(*result));
 	auto &current_chunk = result->current_chunk;
 	if (current_chunk.Valid()) {
 		// We start by scanning the non-finished current chunk

package/src/duckdb/src/function/table/read_csv.cpp

@@ -424,6 +424,7 @@ bool ParallelCSVGlobalState::Finished() {
 
 void ParallelCSVGlobalState::Verify() {
 	// All threads are done, we run some magic sweet verification code
+	lock_guard<mutex> parallel_lock(main_mutex);
 	if (running_threads == 0) {
 		D_ASSERT(tuple_end.size() == tuple_start.size());
 		for (idx_t i = 0; i < tuple_start.size(); i++) {

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.8.1-dev416"
+#define DUCKDB_VERSION "0.8.1-dev477"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "9d5158ccd2"
+#define DUCKDB_SOURCE_ID "3dfd7664a8"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/common/arrow/arrow_appender.hpp

@@ -19,7 +19,7 @@ struct ArrowAppendData;
 //! The ArrowAppender class can be used to incrementally construct an arrow array by appending data chunks into it
 class ArrowAppender {
 public:
-	DUCKDB_API ArrowAppender(vector<LogicalType> types, idx_t initial_capacity, ArrowOptions options = ArrowOptions());
+	DUCKDB_API ArrowAppender(vector<LogicalType> types, idx_t initial_capacity, ArrowOptions options);
 	DUCKDB_API ~ArrowAppender();
 
 	//! Append a data chunk to the underlying arrow array

package/src/duckdb/src/include/duckdb/common/arrow/arrow_converter.hpp

@@ -18,9 +18,8 @@ namespace duckdb {
 
 struct ArrowConverter {
 	DUCKDB_API static void ToArrowSchema(ArrowSchema *out_schema, const vector<LogicalType> &types,
-	                                     const vector<string> &names, const string &config_timezone,
-	                                     ArrowOptions options = ArrowOptions());
-	DUCKDB_API static void ToArrowArray(DataChunk &input, ArrowArray *out_array, ArrowOptions options = ArrowOptions());
+	                                     const vector<string> &names, const ArrowOptions &options);
+	DUCKDB_API static void ToArrowArray(DataChunk &input, ArrowArray *out_array, ArrowOptions options);
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/arrow/arrow_options.hpp

@@ -13,6 +13,13 @@ namespace duckdb {
 enum ArrowOffsetSize { REGULAR, LARGE };
 
 struct ArrowOptions {
-	ArrowOffsetSize offset_size = ArrowOffsetSize::LARGE;
+	explicit ArrowOptions(ArrowOffsetSize offset_size_p) : offset_size(offset_size_p) {
+	}
+	ArrowOptions(ArrowOffsetSize offset_size_p, string timezone_p) : offset_size(offset_size_p), time_zone(timezone_p) {
+	}
+	ArrowOptions() {
+	}
+	ArrowOffsetSize offset_size = ArrowOffsetSize::REGULAR;
+	string time_zone = "UTC";
 };
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/common/arrow/result_arrow_wrapper.hpp

@@ -21,7 +21,6 @@ public:
 	idx_t batch_size;
 	vector<LogicalType> column_types;
 	vector<string> column_names;
-	string timezone_config;
 
 private:
 	static int MyStreamGetSchema(struct ArrowArrayStream *stream, struct ArrowSchema *out);

package/src/duckdb/src/include/duckdb/common/bit_utils.hpp

@@ -112,35 +112,29 @@ struct CountZeros<uint64_t> {
 template <>
 struct CountZeros<hugeint_t> {
 	inline static int Leading(hugeint_t value) {
-		if (value == 0) {
+		const uint64_t upper = (uint64_t)value.upper;
+		const uint64_t lower = value.lower;
+
+		if (upper) {
+			return __builtin_clzll(upper);
+		} else if (lower) {
+			return 64 + __builtin_clzll(lower);
+		} else {
 			return 128;
 		}
-
-		uint64_t upper = (uint64_t)value.upper;
-		uint64_t lower = value.lower;
-
-		int res = __builtin_clzll(upper);
-		if (res == 64) {
-			res += __builtin_clzll(lower);
-		}
-
-		return res;
 	}
 
 	inline static int Trailing(hugeint_t value) {
-		if (value == 0) {
+		const uint64_t upper = (uint64_t)value.upper;
+		const uint64_t lower = value.lower;
+
+		if (lower) {
+			return __builtin_ctzll(lower);
+		} else if (upper) {
+			return 64 + __builtin_ctzll(upper);
+		} else {
 			return 128;
 		}
-
-		uint64_t upper = (uint64_t)value.upper;
-		uint64_t lower = value.lower;
-
-		int res = __builtin_ctzll(lower);
-		if (res == 64) {
-			res += __builtin_ctzll(upper);
-		}
-
-		return res;
 	}
 };
 

package/src/duckdb/src/include/duckdb/main/capi/capi_internal.hpp

@@ -13,6 +13,8 @@
 #include "duckdb/common/types.hpp"
 #include "duckdb/common/types/data_chunk.hpp"
 #include "duckdb/main/appender.hpp"
+#include "duckdb/common/arrow/arrow_options.hpp"
+
 #include <cstring>
 #include <cassert>
 
@@ -46,7 +48,7 @@ struct PendingStatementWrapper {
 struct ArrowResultWrapper {
 	unique_ptr<MaterializedQueryResult> result;
 	unique_ptr<DataChunk> current_chunk;
-	string timezone_config;
+	ArrowOptions options;
 };
 
 struct AppenderWrapper {

package/src/duckdb/src/include/duckdb/main/config.hpp

@@ -28,6 +28,7 @@
 #include "duckdb/optimizer/optimizer_extension.hpp"
 #include "duckdb/parser/parser_extension.hpp"
 #include "duckdb/planner/operator_extension.hpp"
+#include "duckdb/common/arrow/arrow_options.hpp"
 
 namespace duckdb {
 class BufferPool;
@@ -135,6 +136,8 @@ struct DBConfigOptions {
 	WindowAggregationMode window_mode = WindowAggregationMode::WINDOW;
 	//! Whether or not preserving insertion order should be preserved
 	bool preserve_insertion_order = true;
+	//! Whether Arrow Arrays use Large or Regular buffers
+	ArrowOffsetSize arrow_offset_size = ArrowOffsetSize::REGULAR;
 	//! Database configuration variables as controlled by SET
 	case_insensitive_map_t<Value> set_variables;
 	//! Database configuration variable default values;

package/src/duckdb/src/include/duckdb/main/database_manager.hpp

@@ -44,6 +44,7 @@ public:
 	//! Returns a reference to the system catalog
 	Catalog &GetSystemCatalog();
 	static const string &GetDefaultDatabase(ClientContext &context);
+	void SetDefaultDatabase(ClientContext &context, const string &new_value);
 
 	optional_ptr<AttachedDatabase> GetDatabaseFromPath(ClientContext &context, const string &path);
 	vector<reference<AttachedDatabase>> GetDatabases(ClientContext &context);

package/src/duckdb/src/include/duckdb/main/query_result.hpp

@@ -12,6 +12,7 @@
 #include "duckdb/common/types/data_chunk.hpp"
 #include "duckdb/common/winapi.hpp"
 #include "duckdb/common/preserved_error.hpp"
+#include "duckdb/common/arrow/arrow_options.hpp"
 
 namespace duckdb {
 struct BoxRendererConfig;
@@ -20,7 +21,11 @@ enum class QueryResultType : uint8_t { MATERIALIZED_RESULT, STREAM_RESULT, PENDI
 
 //! A set of properties from the client context that can be used to interpret the query result
 struct ClientProperties {
+	ClientProperties(string time_zone_p, ArrowOffsetSize arrow_offset_size_p)
+	    : time_zone(std::move(time_zone_p)), arrow_offset_size(arrow_offset_size_p) {
+	}
 	string time_zone;
+	ArrowOffsetSize arrow_offset_size;
 };
 
 class BaseQueryResult {
@@ -141,6 +146,7 @@ public:
 		}
 	}
 
+	static ArrowOptions GetArrowOptions(QueryResult &query_result);
 	static string GetConfigTimezone(QueryResult &query_result);
 
 private:

package/src/duckdb/src/include/duckdb/main/settings.hpp

@@ -397,6 +397,16 @@ struct PreserveInsertionOrder {
 	static Value GetSetting(ClientContext &context);
 };
 
+struct ExportLargeBufferArrow {
+	static constexpr const char *Name = "arrow_large_buffer_size";
+	static constexpr const char *Description =
+	    "If arrow buffers for strings, blobs, uuids and bits should be exported using large buffers";
+	static constexpr const LogicalTypeId InputType = LogicalTypeId::BOOLEAN;
+	static void SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &parameter);
+	static void ResetGlobal(DatabaseInstance *db, DBConfig &config);
+	static Value GetSetting(ClientContext &context);
+};
+
 struct ProfilerHistorySize {
 	static constexpr const char *Name = "profiler_history_size";
 	static constexpr const char *Description = "Sets the profiler history size";

package/src/duckdb/src/include/duckdb/parser/tableref/pivotref.hpp

@@ -93,9 +93,9 @@ public:
 
 	unique_ptr<TableRef> Copy() override;
 
-	//! Serializes a blob into a JoinRef
+	//! Serializes a blob into a PivotRef
 	void Serialize(FieldWriter &serializer) const override;
-	//! Deserializes a blob back into a JoinRef
+	//! Deserializes a blob back into a PivotRef
 	static unique_ptr<TableRef> Deserialize(FieldReader &source);
 
 	void FormatSerialize(FormatSerializer &serializer) const override;

package/src/duckdb/src/main/capi/arrow-c.cpp

@@ -25,7 +25,7 @@ duckdb_state duckdb_query_arrow_schema(duckdb_arrow result, duckdb_arrow_schema
 	}
 	auto wrapper = reinterpret_cast<ArrowResultWrapper *>(result);
 	ArrowConverter::ToArrowSchema((ArrowSchema *)*out_schema, wrapper->result->types, wrapper->result->names,
-	                              wrapper->timezone_config);
+	                              wrapper->options);
 	return DuckDBSuccess;
 }
 
@@ -41,7 +41,7 @@ duckdb_state duckdb_query_arrow_array(duckdb_arrow result, duckdb_arrow_array *o
 	if (!wrapper->current_chunk || wrapper->current_chunk->size() == 0) {
 		return DuckDBSuccess;
 	}
-	ArrowConverter::ToArrowArray(*wrapper->current_chunk, reinterpret_cast<ArrowArray *>(*out_array));
+	ArrowConverter::ToArrowArray(*wrapper->current_chunk, reinterpret_cast<ArrowArray *>(*out_array), wrapper->options);
 	return DuckDBSuccess;
 }
 
@@ -96,9 +96,9 @@ duckdb_state duckdb_execute_prepared_arrow(duckdb_prepared_statement prepared_st
 	auto arrow_wrapper = new ArrowResultWrapper();
 	if (wrapper->statement->context->config.set_variables.find("TimeZone") ==
 	    wrapper->statement->context->config.set_variables.end()) {
-		arrow_wrapper->timezone_config = "UTC";
+		arrow_wrapper->options.time_zone = "UTC";
 	} else {
-		arrow_wrapper->timezone_config =
+		arrow_wrapper->options.time_zone =
 		    wrapper->statement->context->config.set_variables["TimeZone"].GetValue<std::string>();
 	}
 

package/src/duckdb/src/main/client_context.cpp

@@ -319,6 +319,8 @@ shared_ptr<PreparedStatementData> ClientContext::CreatePreparedStatement(ClientC
 			planner.parameter_data.emplace_back(value);
 		}
 	}
+
+	client_data->http_state = make_shared<HTTPState>();
 	planner.CreatePlan(std::move(statement));
 	D_ASSERT(planner.plan || !planner.properties.bound_all_parameters);
 	profiler.EndPhase();
@@ -1153,9 +1155,8 @@ ParserOptions ClientContext::GetParserOptions() const {
 }
 
 ClientProperties ClientContext::GetClientProperties() const {
-	ClientProperties properties;
-	properties.time_zone = ClientConfig::GetConfig(*this).ExtractTimezone();
-	return properties;
+	auto client_context = ClientConfig::GetConfig(*this);
+	return {client_context.ExtractTimezone(), db->config.options.arrow_offset_size};
 }
 
 bool ClientContext::ExecutionIsFinished() {

package/src/duckdb/src/main/config.cpp

@@ -105,6 +105,7 @@ static ConfigurationOption internal_options[] = {DUCKDB_GLOBAL(AccessModeSetting
                                                  DUCKDB_GLOBAL(TempDirectorySetting),
                                                  DUCKDB_GLOBAL(ThreadsSetting),
                                                  DUCKDB_GLOBAL(UsernameSetting),
+                                                 DUCKDB_GLOBAL(ExportLargeBufferArrow),
                                                  DUCKDB_GLOBAL_ALIAS("user", UsernameSetting),
                                                  DUCKDB_GLOBAL_ALIAS("wal_autocheckpoint", CheckpointThresholdSetting),
                                                  DUCKDB_GLOBAL_ALIAS("worker_threads", ThreadsSetting),

package/src/duckdb/src/main/database_manager.cpp

@@ -87,6 +87,22 @@ const string &DatabaseManager::GetDefaultDatabase(ClientContext &context) {
 	return default_entry.catalog;
 }
 
+// LCOV_EXCL_START
+void DatabaseManager::SetDefaultDatabase(ClientContext &context, const string &new_value) {
+	auto db_entry = GetDatabase(context, new_value);
+
+	if (!db_entry) {
+		throw InternalException("Database \"%s\" not found", new_value);
+	} else if (db_entry->IsTemporary()) {
+		throw InternalException("Cannot set the default database to a temporary database");
+	} else if (db_entry->IsSystem()) {
+		throw InternalException("Cannot set the default database to a system database");
+	}
+
+	default_database = new_value;
+}
+// LCOV_EXCL_STOP
+
 vector<reference<AttachedDatabase>> DatabaseManager::GetDatabases(ClientContext &context) {
 	vector<reference<AttachedDatabase>> result;
 	databases->Scan(context, [&](CatalogEntry &entry) { result.push_back(entry.Cast<AttachedDatabase>()); });

package/src/duckdb/src/main/query_result.cpp

@@ -3,7 +3,6 @@
 #include "duckdb/common/vector.hpp"
 #include "duckdb/main/client_context.hpp"
 #include "duckdb/common/box_renderer.hpp"
-
 namespace duckdb {
 
 BaseQueryResult::BaseQueryResult(QueryResultType type, StatementType statement_type, StatementProperties properties_p,
@@ -71,7 +70,8 @@ idx_t CurrentChunk::RemainingSize() {
 	return data_chunk->size() - position;
 }
 
-QueryResult::QueryResult(QueryResultType type, PreservedError error) : BaseQueryResult(type, std::move(error)) {
+QueryResult::QueryResult(QueryResultType type, PreservedError error)
+    : BaseQueryResult(type, std::move(error)), client_properties("UTC", ArrowOffsetSize::REGULAR) {
 }
 
 QueryResult::~QueryResult() {
@@ -164,6 +164,10 @@ string QueryResult::HeaderToString() {
 	return result;
 }
 
+ArrowOptions QueryResult::GetArrowOptions(QueryResult &query_result) {
+	return {query_result.client_properties.arrow_offset_size, query_result.client_properties.time_zone};
+}
+
 string QueryResult::GetConfigTimezone(QueryResult &query_result) {
 	return query_result.client_properties.time_zone;
 }

package/src/duckdb/src/main/settings/settings.cpp

@@ -895,6 +895,25 @@ Value PreserveInsertionOrder::GetSetting(ClientContext &context) {
 	return Value::BOOLEAN(config.options.preserve_insertion_order);
 }
 
+//===--------------------------------------------------------------------===//
+// ExportLargeBufferArrow
+//===--------------------------------------------------------------------===//
+void ExportLargeBufferArrow::SetGlobal(DatabaseInstance *db, DBConfig &config, const Value &input) {
+	auto export_large_buffers_arrow = input.GetValue<bool>();
+
+	config.options.arrow_offset_size = export_large_buffers_arrow ? ArrowOffsetSize::LARGE : ArrowOffsetSize::REGULAR;
+}
+
+void ExportLargeBufferArrow::ResetGlobal(DatabaseInstance *db, DBConfig &config) {
+	config.options.arrow_offset_size = DBConfig().options.arrow_offset_size;
+}
+
+Value ExportLargeBufferArrow::GetSetting(ClientContext &context) {
+	auto &config = DBConfig::GetConfig(context);
+	bool export_large_buffers_arrow = config.options.arrow_offset_size == ArrowOffsetSize::LARGE;
+	return Value::BOOLEAN(export_large_buffers_arrow);
+}
+
 //===--------------------------------------------------------------------===//
 // Profiler History Size
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/parser/parser.cpp

@@ -1,18 +1,17 @@
 #include "duckdb/parser/parser.hpp"
 
-#include "duckdb/parser/transformer.hpp"
 #include "duckdb/parser/parsed_data/create_table_info.hpp"
+#include "duckdb/parser/parser_extension.hpp"
+#include "duckdb/parser/query_error_context.hpp"
+#include "duckdb/parser/query_node/select_node.hpp"
 #include "duckdb/parser/statement/create_statement.hpp"
 #include "duckdb/parser/statement/extension_statement.hpp"
 #include "duckdb/parser/statement/select_statement.hpp"
 #include "duckdb/parser/statement/update_statement.hpp"
-#include "duckdb/parser/query_node/select_node.hpp"
 #include "duckdb/parser/tableref/expressionlistref.hpp"
-#include "postgres_parser.hpp"
-#include "duckdb/parser/query_error_context.hpp"
-#include "duckdb/parser/parser_extension.hpp"
-
+#include "duckdb/parser/transformer.hpp"
 #include "parser/parser.hpp"
+#include "postgres_parser.hpp"
 
 namespace duckdb {
 
@@ -124,6 +123,29 @@ end:
 	return ReplaceUnicodeSpaces(query_str, new_query, unicode_spaces);
 }
 
+vector<string> SplitQueryStringIntoStatements(const string &query) {
+	// Break sql string down into sql statements using the tokenizer
+	vector<string> query_statements;
+	auto tokens = Parser::Tokenize(query);
+	auto next_statement_start = 0;
+	for (idx_t i = 1; i < tokens.size(); ++i) {
+		auto &t_prev = tokens[i - 1];
+		auto &t = tokens[i];
+		if (t_prev.type == SimplifiedTokenType::SIMPLIFIED_TOKEN_OPERATOR) {
+			// LCOV_EXCL_START
+			for (idx_t c = t_prev.start; c <= t.start; ++c) {
+				if (query.c_str()[c] == ';') {
+					query_statements.emplace_back(query.substr(next_statement_start, t.start - next_statement_start));
+					next_statement_start = tokens[i].start;
+				}
+			}
+			// LCOV_EXCL_STOP
+		}
+	}
+	query_statements.emplace_back(query.substr(next_statement_start, query.size() - next_statement_start));
+	return query_statements;
+}
+
 void Parser::ParseQuery(const string &query) {
 	Transformer transformer(options);
 	string parser_error;
@@ -138,39 +160,77 @@ void Parser::ParseQuery(const string &query) {
 	}
 	{
 		PostgresParser::SetPreserveIdentifierCase(options.preserve_identifier_case);
-		PostgresParser parser;
-		parser.Parse(query);
-		if (parser.success) {
-			if (!parser.parse_tree) {
-				// empty statement
-				return;
-			}
-
-			// if it succeeded, we transform the Postgres parse tree into a list of
-			// SQLStatements
-			transformer.TransformParseTree(parser.parse_tree, statements);
-		} else {
-			parser_error = QueryErrorContext::Format(query, parser.error_message, parser.error_location - 1);
-		}
-	}
-	if (!parser_error.empty()) {
-		if (options.extensions) {
-			for (auto &ext : *options.extensions) {
-				D_ASSERT(ext.parse_function);
-				auto result = ext.parse_function(ext.parser_info.get(), query);
-				if (result.type == ParserExtensionResultType::PARSE_SUCCESSFUL) {
-					auto statement = make_uniq<ExtensionStatement>(ext, std::move(result.parse_data));
-					statement->stmt_length = query.size();
-					statement->stmt_location = 0;
-					statements.push_back(std::move(statement));
+		bool parsing_succeed = false;
+		// Creating a new scope to prevent multiple PostgresParser destructors being called
+		// which led to some memory issues
+		{
+			PostgresParser parser;
+			parser.Parse(query);
+			if (parser.success) {
+				if (!parser.parse_tree) {
+					// empty statement
 					return;
 				}
-				if (result.type == ParserExtensionResultType::DISPLAY_EXTENSION_ERROR) {
-					throw ParserException(result.error);
+
+				// if it succeeded, we transform the Postgres parse tree into a list of
+				// SQLStatements
+				transformer.TransformParseTree(parser.parse_tree, statements);
+				parsing_succeed = true;
+			} else {
+				parser_error = QueryErrorContext::Format(query, parser.error_message, parser.error_location - 1);
+			}
+		}
+		// If DuckDB fails to parse the entire sql string, break the string down into individual statements
+		// using ';' as the delimiter so that parser extensions can parse the statement
+		if (parsing_succeed) {
+			// no-op
+			// return here would require refactoring into another function. o.w. will just no-op in order to run wrap up
+			// code at the end of this function
+		} else if (!options.extensions || options.extensions->empty()) {
+			throw ParserException(parser_error);
+		} else {
+			// split sql string into statements and re-parse using extension
+			auto query_statements = SplitQueryStringIntoStatements(query);
+			for (auto const &query_statement : query_statements) {
+				PostgresParser another_parser;
+				another_parser.Parse(query_statement);
+				// LCOV_EXCL_START
+				// first see if DuckDB can parse this individual query statement
+				if (another_parser.success) {
+					if (!another_parser.parse_tree) {
+						// empty statement
+						continue;
+					}
+					transformer.TransformParseTree(another_parser.parse_tree, statements);
+				} else {
+					// let extensions parse the statement which DuckDB failed to parse
+					bool parsed_single_statement = false;
+					for (auto &ext : *options.extensions) {
+						D_ASSERT(!parsed_single_statement);
+						D_ASSERT(ext.parse_function);
+						auto result = ext.parse_function(ext.parser_info.get(), query_statement);
+						if (result.type == ParserExtensionResultType::PARSE_SUCCESSFUL) {
+							auto statement = make_uniq<ExtensionStatement>(ext, std::move(result.parse_data));
+							statement->stmt_length = query_statement.size();
+							statement->stmt_location = 0;
+							statements.push_back(std::move(statement));
+							parsed_single_statement = true;
+							break;
+						} else if (result.type == ParserExtensionResultType::DISPLAY_EXTENSION_ERROR) {
+							throw ParserException(result.error);
+						} else {
+							// We move to the next one!
+						}
+					}
+					if (!parsed_single_statement) {
+						parser_error = QueryErrorContext::Format(query, another_parser.error_message,
+						                                         another_parser.error_location - 1);
+						throw ParserException(parser_error);
+					}
 				}
+				// LCOV_EXCL_STOP
 			}
 		}
-		throw ParserException(parser_error);
 	}
 	if (!statements.empty()) {
 		auto &last_statement = statements.back();

package/src/duckdb/src/planner/binder/tableref/bind_table_function.cpp

@@ -97,6 +97,8 @@ bool Binder::BindTableFunctionParameters(TableFunctionCatalogEntry &table_functi
 			subquery = make_uniq<BoundSubqueryRef>(std::move(binder), std::move(node));
 			seen_subquery = true;
 			arguments.emplace_back(LogicalTypeId::TABLE);
+			parameters.emplace_back(
+			    Value(LogicalType::INVALID)); // this is a dummy value so the lengths of arguments and parameter match
 			continue;
 		}
 

package/test/extension.test.ts

@@ -3,6 +3,10 @@ import {Database, DuckDbError, HttpError, TableData} from '..';
 import * as fs from 'fs';
 import * as assert from 'assert';
 import * as path from 'path';
+import chaiAsPromised from 'chai-as-promised';
+import chai, {expect} from "chai";
+
+chai.use(chaiAsPromised);
 
 const extension_base_path = "../../../build/release/extension";
 
@@ -26,6 +30,13 @@ function isHTTPException(err: DuckDbError): err is HttpError {
 
 // Note: test will pass on http request failing due to connection issues.
 const test_httpfs = async function (db: duckdb.Database) {
+    const promise = new Promise<void>((resolve, reject) =>
+        db.exec(`SELECT *
+                 FROM parquet_scan('http://localhost:1234/whatever.parquet')`, function (err: DuckDbError | null) {
+            err ? reject(err) : resolve()
+        }));
+    await chai.assert.isRejected(promise, 'IO Error: Connection error for HTTP HEAD');
+
     await new Promise<void>((resolve, reject) => db.all("SELECT id, first_name, last_name FROM PARQUET_SCAN('https://raw.githubusercontent.com/cwida/duckdb/master/data/parquet-testing/userdata1.parquet') LIMIT 3;", function (err: null | Error, rows: TableData) {
         if (err) {
             if (err.message.startsWith("Unable to connect to URL")) {