duckdb 0.8.2-dev2320.0 → 0.8.2-dev2399.0

package/binding.gyp

@@ -244,18 +244,18 @@
                 "src/duckdb/third_party/zstd/compress/zstd_lazy.cpp", 
                 "src/duckdb/third_party/zstd/compress/zstd_ldm.cpp", 
                 "src/duckdb/third_party/zstd/compress/zstd_opt.cpp", 
-                "src/duckdb/extension/icu/./icu_extension.cpp", 
-                "src/duckdb/extension/icu/./icu-datetrunc.cpp", 
-                "src/duckdb/extension/icu/./icu-datepart.cpp", 
                 "src/duckdb/extension/icu/./icu-strptime.cpp", 
+                "src/duckdb/extension/icu/./icu-datepart.cpp", 
+                "src/duckdb/extension/icu/./icu-datesub.cpp", 
                 "src/duckdb/extension/icu/./icu-table-range.cpp", 
-                "src/duckdb/extension/icu/./icu-makedate.cpp", 
-                "src/duckdb/extension/icu/./icu-datefunc.cpp", 
-                "src/duckdb/extension/icu/./icu-list-range.cpp", 
+                "src/duckdb/extension/icu/./icu-datetrunc.cpp", 
                 "src/duckdb/extension/icu/./icu-timebucket.cpp", 
                 "src/duckdb/extension/icu/./icu-dateadd.cpp", 
+                "src/duckdb/extension/icu/./icu-list-range.cpp", 
                 "src/duckdb/extension/icu/./icu-timezone.cpp", 
-                "src/duckdb/extension/icu/./icu-datesub.cpp", 
+                "src/duckdb/extension/icu/./icu-datefunc.cpp", 
+                "src/duckdb/extension/icu/./icu_extension.cpp", 
+                "src/duckdb/extension/icu/./icu-makedate.cpp", 
                 "src/duckdb/ub_extension_icu_third_party_icu_common.cpp", 
                 "src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp", 
                 "src/duckdb/extension/icu/third_party/icu/stubdata/stubdata.cpp", 

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.8.2-dev2320.0",
+  "version": "0.8.2-dev2399.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/extension/icu/icu-datefunc.cpp

@@ -5,6 +5,8 @@
 #include "duckdb/common/operator/multiply.hpp"
 #include "duckdb/common/types/timestamp.hpp"
 
+#include "unicode/ucal.h"
+
 namespace duckdb {
 
 ICUDateFunc::BindData::BindData(const BindData &other)
@@ -34,6 +36,13 @@ ICUDateFunc::BindData::BindData(ClientContext &context) {
 	if (U_FAILURE(success)) {
 		throw Exception("Unable to create ICU calendar.");
 	}
+
+	//	Postgres always assumes times are given in the proleptic Gregorian calendar.
+	//	ICU defaults to the Gregorian change in 1582, so we reset the change to the minimum date
+	//	so that all dates are proleptic Gregorian.
+	//	The only error here is if we have a non-Gregorian calendar,
+	//	and we just ignore that and hope for the best...
+	ucal_setGregorianChange((UCalendar *)calendar.get(), U_DATE_MIN, &success); // NOLINT
 }
 
 bool ICUDateFunc::BindData::Equals(const FunctionData &other_p) const {

package/src/duckdb/extension/icu/icu-datepart.cpp

@@ -109,11 +109,8 @@ struct ICUDatePart : public ICUDateFunc {
 
 	static int64_t ExtractEpoch(icu::Calendar *calendar, const uint64_t micros) {
 		UErrorCode status = U_ZERO_ERROR;
-		auto millis = calendar->getTime(status);
-		millis += ExtractField(calendar, UCAL_ZONE_OFFSET);
-		millis += ExtractField(calendar, UCAL_DST_OFFSET);
 		//	Truncate
-		return millis / Interval::MSECS_PER_SEC;
+		return calendar->getTime(status) / Interval::MSECS_PER_SEC;
 	}
 
 	static int64_t ExtractTimezone(icu::Calendar *calendar, const uint64_t micros) {
@@ -200,7 +197,12 @@ struct ICUDatePart : public ICUDateFunc {
 
 		calendar->set(UCAL_DATE, dd);
 
-		return Date::EpochToDate(ExtractEpoch(calendar, 0));
+		//	Offset to UTC
+		auto millis = calendar->getTime(status);
+		millis += ExtractField(calendar, UCAL_ZONE_OFFSET);
+		millis += ExtractField(calendar, UCAL_DST_OFFSET);
+
+		return Date::EpochToDate(millis / Interval::MSECS_PER_SEC);
 	}
 
 	static string_t MonthName(icu::Calendar *calendar, const uint64_t micros) {

package/src/duckdb/extension/icu/icu-strptime.cpp

@@ -273,26 +273,7 @@ struct ICUStrptime : public ICUDateFunc {
 				    }
 
 				    // Now get the parts in the given time zone
-				    date_t d;
-				    dtime_t t;
-				    Timestamp::Convert(result, d, t);
-
-				    int32_t data[7];
-				    Date::Convert(d, data[0], data[1], data[2]);
-				    calendar->set(UCAL_EXTENDED_YEAR, data[0]); // strptime doesn't understand eras
-				    calendar->set(UCAL_MONTH, data[1] - 1);
-				    calendar->set(UCAL_DATE, data[2]);
-
-				    Time::Convert(t, data[3], data[4], data[5], data[6]);
-				    calendar->set(UCAL_HOUR_OF_DAY, data[3]);
-				    calendar->set(UCAL_MINUTE, data[4]);
-				    calendar->set(UCAL_SECOND, data[5]);
-
-				    int32_t millis = data[6] / Interval::MICROS_PER_MSEC;
-				    uint64_t micros = data[6] % Interval::MICROS_PER_MSEC;
-				    calendar->set(UCAL_MILLISECOND, millis);
-
-				    result = GetTime(calendar, micros);
+				    result = FromNaive(calendar, result);
 			    }
 
 			    return result;

package/src/duckdb/extension/parquet/parquet_writer.cpp

@@ -297,6 +297,7 @@ void ParquetWriter::PrepareRowGroup(ColumnDataCollection &buffer, PreparedRowGro
 	// set up a new row group for this chunk collection
 	auto &row_group = result.row_group;
 	row_group.num_rows = buffer.Count();
+	row_group.total_byte_size = buffer.SizeInBytes();
 	row_group.__isset.file_offset = true;
 
 	auto &states = result.states;

package/src/duckdb/src/common/adbc/adbc.cpp

@@ -202,14 +202,15 @@ AdbcStatusCode ConnectionGetTableSchema(struct AdbcConnection *connection, const
 		SetError(error, "Connection is not set");
 		return ADBC_STATUS_INVALID_ARGUMENT;
 	}
+	if (db_schema == nullptr) {
+		// if schema is not set, we use the default schema
+		db_schema = "main";
+	}
 	if (catalog != nullptr && strlen(catalog) > 0) {
 		// In DuckDB this is the name of the database, not sure what's the expected functionality here, so for now,
 		// scream.
 		SetError(error, "Catalog Name is not used in DuckDB. It must be set to nullptr or an empty string");
 		return ADBC_STATUS_NOT_IMPLEMENTED;
-	} else if (db_schema == nullptr) {
-		SetError(error, "AdbcConnectionGetTableSchema: must provide db_schema");
-		return ADBC_STATUS_INVALID_ARGUMENT;
 	} else if (table_name == nullptr) {
 		SetError(error, "AdbcConnectionGetTableSchema: must provide table_name");
 		return ADBC_STATUS_INVALID_ARGUMENT;
@@ -686,6 +687,10 @@ AdbcStatusCode QueryInternal(struct AdbcConnection *connection, struct ArrowArra
 AdbcStatusCode ConnectionGetObjects(struct AdbcConnection *connection, int depth, const char *catalog,
                                     const char *db_schema, const char *table_name, const char **table_type,
                                     const char *column_name, struct ArrowArrayStream *out, struct AdbcError *error) {
+	if (depth != 0) {
+		SetError(error, "Depth parameter not yet supported");
+		return ADBC_STATUS_NOT_IMPLEMENTED;
+	}
 	if (catalog != nullptr) {
 		if (strcmp(catalog, "duckdb") == 0) {
 			SetError(error, "catalog must be NULL or 'duckdb'");

package/src/duckdb/src/common/arrow/arrow_appender.cpp

@@ -14,7 +14,7 @@ namespace duckdb {
 // ArrowAppender
 //===--------------------------------------------------------------------===//
 
-ArrowAppender::ArrowAppender(vector<LogicalType> types_p, idx_t initial_capacity, ArrowOptions options)
+ArrowAppender::ArrowAppender(vector<LogicalType> types_p, idx_t initial_capacity, ClientProperties options)
     : types(std::move(types_p)) {
 	for (auto &type : types) {
 		auto entry = ArrowAppender::InitializeChild(type, initial_capacity, options);
@@ -179,14 +179,14 @@ static void InitializeFunctionPointers(ArrowAppendData &append_data, const Logic
 	case LogicalTypeId::VARCHAR:
 	case LogicalTypeId::BLOB:
 	case LogicalTypeId::BIT:
-		if (append_data.options.offset_size == ArrowOffsetSize::LARGE) {
+		if (append_data.options.arrow_offset_size == ArrowOffsetSize::LARGE) {
 			InitializeAppenderForType<ArrowVarcharData<string_t>>(append_data);
 		} else {
 			InitializeAppenderForType<ArrowVarcharData<string_t, ArrowVarcharConverter, uint32_t>>(append_data);
 		}
 		break;
 	case LogicalTypeId::UUID:
-		if (append_data.options.offset_size == ArrowOffsetSize::LARGE) {
+		if (append_data.options.arrow_offset_size == ArrowOffsetSize::LARGE) {
 			InitializeAppenderForType<ArrowVarcharData<hugeint_t, ArrowUUIDConverter>>(append_data);
 		} else {
 			InitializeAppenderForType<ArrowVarcharData<hugeint_t, ArrowUUIDConverter, uint32_t>>(append_data);
@@ -228,7 +228,7 @@ static void InitializeFunctionPointers(ArrowAppendData &append_data, const Logic
 }
 
 unique_ptr<ArrowAppendData> ArrowAppender::InitializeChild(const LogicalType &type, idx_t capacity,
-                                                           ArrowOptions &options) {
+                                                           ClientProperties &options) {
 	auto result = make_uniq<ArrowAppendData>(options);
 	InitializeFunctionPointers(*result, type);
 

package/src/duckdb/src/common/arrow/arrow_converter.cpp

@@ -15,12 +15,21 @@
 
 namespace duckdb {
 
-void ArrowConverter::ToArrowArray(DataChunk &input, ArrowArray *out_array, ArrowOptions options) {
+void ArrowConverter::ToArrowArray(DataChunk &input, ArrowArray *out_array, ClientProperties options) {
 	ArrowAppender appender(input.GetTypes(), input.size(), std::move(options));
 	appender.Append(input, 0, input.size(), input.size());
 	*out_array = appender.Finalize();
 }
 
+unsafe_unique_array<char> AddName(const string &name) {
+	auto name_ptr = make_unsafe_uniq_array<char>(name.size() + 1);
+	for (size_t i = 0; i < name.size(); i++) {
+		name_ptr[i] = name[i];
+	}
+	name_ptr[name.size()] = '\0';
+	return name_ptr;
+}
+
 //===--------------------------------------------------------------------===//
 // Arrow Schema
 //===--------------------------------------------------------------------===//
@@ -45,24 +54,26 @@ static void ReleaseDuckDBArrowSchema(ArrowSchema *schema) {
 	delete holder;
 }
 
-void InitializeChild(ArrowSchema &child, const string &name = "") {
+void InitializeChild(ArrowSchema &child, DuckDBArrowSchemaHolder &root_holder, const string &name = "") {
 	//! Child is cleaned up by parent
 	child.private_data = nullptr;
 	child.release = ReleaseDuckDBArrowSchema;
 
 	//! Store the child schema
 	child.flags = ARROW_FLAG_NULLABLE;
-	child.name = name.c_str();
+	root_holder.owned_type_names.push_back(AddName(name));
+
+	child.name = root_holder.owned_type_names.back().get();
 	child.n_children = 0;
 	child.children = nullptr;
 	child.metadata = nullptr;
 	child.dictionary = nullptr;
 }
 void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, const LogicalType &type,
-                    const ArrowOptions &options);
+                    const ClientProperties &options);
 
 void SetArrowMapFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, const LogicalType &type,
-                       const ArrowOptions &options) {
+                       const ClientProperties &options) {
 	child.format = "+m";
 	//! Map has one child which is a struct
 	child.n_children = 1;
@@ -70,23 +81,14 @@ void SetArrowMapFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child,
 	root_holder.nested_children.back().resize(1);
 	root_holder.nested_children_ptr.emplace_back();
 	root_holder.nested_children_ptr.back().push_back(&root_holder.nested_children.back()[0]);
-	InitializeChild(root_holder.nested_children.back()[0]);
+	InitializeChild(root_holder.nested_children.back()[0], root_holder);
 	child.children = &root_holder.nested_children_ptr.back()[0];
 	child.children[0]->name = "entries";
 	SetArrowFormat(root_holder, **child.children, ListType::GetChildType(type), options);
 }
 
-unsafe_unique_array<char> AddName(const string &name) {
-	auto name_ptr = make_unsafe_uniq_array<char>(name.size() + 1);
-	for (size_t i = 0; i < name.size(); i++) {
-		name_ptr[i] = name[i];
-	}
-	name_ptr[name.size()] = '\0';
-	return name_ptr;
-}
-
 void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, const LogicalType &type,
-                    const ArrowOptions &options) {
+                    const ClientProperties &options) {
 	switch (type.id()) {
 	case LogicalTypeId::BOOLEAN:
 		child.format = "b";
@@ -126,7 +128,7 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
 		break;
 	case LogicalTypeId::UUID:
 	case LogicalTypeId::VARCHAR:
-		if (options.offset_size == ArrowOffsetSize::LARGE) {
+		if (options.arrow_offset_size == ArrowOffsetSize::LARGE) {
 			child.format = "U";
 		} else {
 			child.format = "u";
@@ -136,7 +138,6 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
 		child.format = "tdD";
 		break;
 	case LogicalTypeId::TIME:
-	case LogicalTypeId::TIME_TZ:
 		child.format = "ttu";
 		break;
 	case LogicalTypeId::TIMESTAMP:
@@ -174,7 +175,7 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
 	}
 	case LogicalTypeId::BLOB:
 	case LogicalTypeId::BIT: {
-		if (options.offset_size == ArrowOffsetSize::LARGE) {
+		if (options.arrow_offset_size == ArrowOffsetSize::LARGE) {
 			child.format = "Z";
 		} else {
 			child.format = "z";
@@ -188,7 +189,7 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
 		root_holder.nested_children.back().resize(1);
 		root_holder.nested_children_ptr.emplace_back();
 		root_holder.nested_children_ptr.back().push_back(&root_holder.nested_children.back()[0]);
-		InitializeChild(root_holder.nested_children.back()[0]);
+		InitializeChild(root_holder.nested_children.back()[0], root_holder);
 		child.children = &root_holder.nested_children_ptr.back()[0];
 		child.children[0]->name = "l";
 		SetArrowFormat(root_holder, **child.children, ListType::GetChildType(type), options);
@@ -208,7 +209,7 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
 		child.children = &root_holder.nested_children_ptr.back()[0];
 		for (size_t type_idx = 0; type_idx < child_types.size(); type_idx++) {
 
-			InitializeChild(*child.children[type_idx]);
+			InitializeChild(*child.children[type_idx], root_holder);
 
 			root_holder.owned_type_names.push_back(AddName(child_types[type_idx].first));
 
@@ -236,7 +237,7 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
 		child.children = &root_holder.nested_children_ptr.back()[0];
 		for (size_t type_idx = 0; type_idx < child_types.size(); type_idx++) {
 
-			InitializeChild(*child.children[type_idx]);
+			InitializeChild(*child.children[type_idx], root_holder);
 
 			root_holder.owned_type_names.push_back(AddName(child_types[type_idx].first));
 
@@ -272,18 +273,18 @@ void SetArrowFormat(DuckDBArrowSchemaHolder &root_holder, ArrowSchema &child, co
 		root_holder.nested_children.back().resize(1);
 		root_holder.nested_children_ptr.emplace_back();
 		root_holder.nested_children_ptr.back().push_back(&root_holder.nested_children.back()[0]);
-		InitializeChild(root_holder.nested_children.back()[0]);
+		InitializeChild(root_holder.nested_children.back()[0], root_holder);
 		child.dictionary = root_holder.nested_children_ptr.back()[0];
 		child.dictionary->format = "u";
 		break;
 	}
 	default:
-		throw InternalException("Unsupported Arrow type " + type.ToString());
+		throw NotImplementedException("Unsupported Arrow type " + type.ToString());
 	}
 }
 
 void ArrowConverter::ToArrowSchema(ArrowSchema *out_schema, const vector<LogicalType> &types,
-                                   const vector<string> &names, const ArrowOptions &options) {
+                                   const vector<string> &names, const ClientProperties &options) {
 	D_ASSERT(out_schema);
 	D_ASSERT(types.size() == names.size());
 	idx_t column_count = types.size();
@@ -310,7 +311,7 @@ void ArrowConverter::ToArrowSchema(ArrowSchema *out_schema, const vector<Logical
 	for (idx_t col_idx = 0; col_idx < column_count; col_idx++) {
 
 		auto &child = root_holder->children[col_idx];
-		InitializeChild(child, names[col_idx]);
+		InitializeChild(child, *root_holder, names[col_idx]);
 		SetArrowFormat(*root_holder, child, types[col_idx], options);
 	}
 

package/src/duckdb/src/common/arrow/arrow_wrapper.cpp

@@ -69,7 +69,7 @@ int ResultArrowArrayStreamWrapper::MyStreamGetSchema(struct ArrowArrayStream *st
 	auto my_stream = reinterpret_cast<ResultArrowArrayStreamWrapper *>(stream->private_data);
 	if (!my_stream->column_types.empty()) {
 		ArrowConverter::ToArrowSchema(out, my_stream->column_types, my_stream->column_names,
-		                              QueryResult::GetArrowOptions(*my_stream->result));
+		                              my_stream->result->client_properties);
 		return 0;
 	}
 
@@ -90,7 +90,7 @@ int ResultArrowArrayStreamWrapper::MyStreamGetSchema(struct ArrowArrayStream *st
 		my_stream->column_names = result.names;
 	}
 	ArrowConverter::ToArrowSchema(out, my_stream->column_types, my_stream->column_names,
-	                              QueryResult::GetArrowOptions(*my_stream->result));
+	                              my_stream->result->client_properties);
 	return 0;
 }
 
@@ -119,7 +119,7 @@ int ResultArrowArrayStreamWrapper::MyStreamGetNext(struct ArrowArrayStream *stre
 	}
 	idx_t result_count;
 	PreservedError error;
-	if (!ArrowUtil::TryFetchChunk(scan_state, result.GetArrowOptions(result), my_stream->batch_size, out, result_count,
+	if (!ArrowUtil::TryFetchChunk(scan_state, result.client_properties, my_stream->batch_size, out, result_count,
 	                              error)) {
 		D_ASSERT(error);
 		my_stream->last_error = error;
@@ -165,7 +165,7 @@ ResultArrowArrayStreamWrapper::ResultArrowArrayStreamWrapper(unique_ptr<QueryRes
 	stream.get_last_error = ResultArrowArrayStreamWrapper::MyStreamGetLastError;
 }
 
-bool ArrowUtil::TryFetchChunk(ChunkScanState &scan_state, ArrowOptions options, idx_t batch_size, ArrowArray *out,
+bool ArrowUtil::TryFetchChunk(ChunkScanState &scan_state, ClientProperties options, idx_t batch_size, ArrowArray *out,
                               idx_t &count, PreservedError &error) {
 	count = 0;
 	ArrowAppender appender(scan_state.Types(), batch_size, std::move(options));
@@ -209,7 +209,7 @@ bool ArrowUtil::TryFetchChunk(ChunkScanState &scan_state, ArrowOptions options,
 	return true;
 }
 
-idx_t ArrowUtil::FetchChunk(ChunkScanState &scan_state, ArrowOptions options, idx_t chunk_size, ArrowArray *out) {
+idx_t ArrowUtil::FetchChunk(ChunkScanState &scan_state, ClientProperties options, idx_t chunk_size, ArrowArray *out) {
 	PreservedError error;
 	idx_t result_count;
 	if (!TryFetchChunk(scan_state, std::move(options), chunk_size, out, result_count, error)) {