duckdb 0.10.1-dev9.0 → 0.10.1

package/.github/workflows/NodeJS.yml

@@ -11,8 +11,9 @@ concurrency:
 
 env:
   GH_TOKEN: ${{ secrets.GH_TOKEN }}
-  AWS_ACCESS_KEY_ID: ${{ secrets.NODE_PRE_GYP_ID }}
-  AWS_SECRET_ACCESS_KEY: ${{ secrets.NODE_PRE_GYP_SECRETACCESSKEY }}
+  AWS_ACCESS_KEY_ID: ${{secrets.S3_DUCKDB_NODE_ID}}
+  AWS_SECRET_ACCESS_KEY: ${{secrets.S3_DUCKDB_NODE_KEY}}
+  AWS_DEFAULT_REGION: us-east-1
 
 jobs:
   set-up-npm:
@@ -25,6 +26,10 @@ jobs:
         with:
           fetch-depth: 0
 
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
       - name: Setup NPM
         shell: bash
         run: ./scripts/node_version.sh upload
@@ -43,7 +48,7 @@ jobs:
     strategy:
       matrix:
         # node.js current support policy to be found at https://github.com/duckdb/duckdb-node/tree/main/#Supported-Node-versions
-        node: [ '16', '17', '18', '19', '20', '21']
+        node: [ '12', '14', '16', '17', '18', '19', '20', '21']
         target_arch: [ x64, arm64 ]
         isRelease:
           - ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}
@@ -76,7 +81,7 @@ jobs:
       # Default Python (3.12) doesn't have support for distutils
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.11'
+          python-version: '3.10'
 
       - name: Update apt
         shell: bash
@@ -107,10 +112,6 @@ jobs:
           npm_config_yes: true
 
       - name: Node ${{ matrix.node }}
-        env:
-          AWS_ACCESS_KEY_ID: ${{secrets.S3_ID}}
-          AWS_SECRET_ACCESS_KEY: ${{secrets.S3_KEY}}
-          AWS_DEFAULT_REGION: us-east-1
         shell: bash
         run: ./scripts/node_build.sh ${{ matrix.node }}
 
@@ -175,10 +176,6 @@ jobs:
           NODE_AUTH_TOKEN: ${{secrets.NODE_AUTH_TOKEN}}
 
       - name: Node ${{ matrix.node }}
-        env:
-          AWS_ACCESS_KEY_ID: ${{secrets.S3_ID}}
-          AWS_SECRET_ACCESS_KEY: ${{secrets.S3_KEY}}
-          AWS_DEFAULT_REGION: us-east-1
         shell: bash
         run: ./scripts/node_build.sh ${{ matrix.node }}
 
@@ -252,9 +249,5 @@ jobs:
           variant: sccache
 
       - name: Node
-        env:
-          AWS_ACCESS_KEY_ID: ${{secrets.S3_ID}}
-          AWS_SECRET_ACCESS_KEY: ${{secrets.S3_KEY}}
-          AWS_DEFAULT_REGION: us-east-1
         shell: bash
         run: ./scripts/node_build_win.sh

package/binding.gyp

@@ -389,6 +389,9 @@
                     {
                         "defines": [
                             "DUCKDB_BUILD_LIBRARY"
+                        ],
+                        "libraries": [
+                            "rstrtmgr.lib", "bcrypt.lib"
                         ]
                     }
                 ]
@@ -411,4 +414,4 @@
             ]
         }
     ]
-}
+}

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.10.1-dev9.0",
+  "version": "0.10.1",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/database.cpp

@@ -141,6 +141,7 @@ static void TaskCompleteCallback(napi_env e, napi_status status, void *data) {
 	duckdb::unique_ptr<TaskHolder> holder((TaskHolder *)data);
 	holder->db->TaskComplete(e);
 	holder->task->DoCallback();
+	napi_delete_async_work(e, holder->request);
 }
 
 void Database::TaskComplete(Napi::Env env) {

package/src/duckdb/extension/icu/icu-strptime.cpp

@@ -246,7 +246,7 @@ struct ICUStrptime : public ICUDateFunc {
 		TailPatch(name, db, types);
 	}
 
-	static bool CastFromVarchar(Vector &source, Vector &result, idx_t count, CastParameters &parameters) {
+	static bool VarcharToTimestampTZ(Vector &source, Vector &result, idx_t count, CastParameters &parameters) {
 		auto &cast_data = parameters.cast_data->Cast<CastData>();
 		auto &info = cast_data.info->Cast<BindData>();
 		CalendarPtr cal(info.calendar->clone());
@@ -260,7 +260,7 @@ struct ICUStrptime : public ICUDateFunc {
 			    bool has_offset = false;
 			    if (!Timestamp::TryConvertTimestampTZ(str, len, result, has_offset, tz)) {
 				    auto msg = Timestamp::ConversionError(string(str, len));
-				    HandleCastError::AssignError(msg, parameters.error_message);
+				    HandleCastError::AssignError(msg, parameters);
 				    mask.SetInvalid(idx);
 			    } else if (!has_offset) {
 				    // Convert parts to a TZ (default or parsed) if no offset was provided
@@ -280,15 +280,56 @@ struct ICUStrptime : public ICUDateFunc {
 		return true;
 	}
 
+	static bool VarcharToTimeTZ(Vector &source, Vector &result, idx_t count, CastParameters &parameters) {
+		auto &cast_data = parameters.cast_data->Cast<CastData>();
+		auto &info = cast_data.info->Cast<BindData>();
+		CalendarPtr cal(info.calendar->clone());
+
+		UnaryExecutor::ExecuteWithNulls<string_t, dtime_tz_t>(
+		    source, result, count, [&](string_t input, ValidityMask &mask, idx_t idx) {
+			    dtime_tz_t result;
+			    const auto str = input.GetData();
+			    const auto len = input.GetSize();
+			    bool has_offset = false;
+			    idx_t pos = 0;
+			    if (!Time::TryConvertTimeTZ(str, len, pos, result, has_offset, false)) {
+				    auto msg = Time::ConversionError(string(str, len));
+				    HandleCastError::AssignError(msg, parameters);
+				    mask.SetInvalid(idx);
+			    } else if (!has_offset) {
+				    // Convert parts to a TZ (default or parsed) if no offset was provided
+				    auto calendar = cal.get();
+
+				    // Extract the offset from the calendar
+				    auto offset = ExtractField(calendar, UCAL_ZONE_OFFSET);
+				    offset += ExtractField(calendar, UCAL_DST_OFFSET);
+				    offset /= Interval::MSECS_PER_SEC;
+
+				    // Apply it to the offset +00 time we parsed.
+				    result = dtime_tz_t(result.time(), offset);
+			    }
+
+			    return result;
+		    });
+		return true;
+	}
+
 	static BoundCastInfo BindCastFromVarchar(BindCastInput &input, const LogicalType &source,
 	                                         const LogicalType &target) {
 		if (!input.context) {
-			throw InternalException("Missing context for VARCHAR to TIMESTAMPTZ cast.");
+			throw InternalException("Missing context for VARCHAR to TIME/TIMESTAMPTZ cast.");
 		}
 
 		auto cast_data = make_uniq<CastData>(make_uniq<BindData>(*input.context));
 
-		return BoundCastInfo(CastFromVarchar, std::move(cast_data));
+		switch (target.id()) {
+		case LogicalTypeId::TIMESTAMP_TZ:
+			return BoundCastInfo(VarcharToTimestampTZ, std::move(cast_data));
+		case LogicalTypeId::TIME_TZ:
+			return BoundCastInfo(VarcharToTimeTZ, std::move(cast_data));
+		default:
+			throw InternalException("Unsupported type for VARCHAR to TIME/TIMESTAMPTZ cast.");
+		}
 	}
 
 	static void AddCasts(DatabaseInstance &db) {
@@ -296,6 +337,7 @@ struct ICUStrptime : public ICUDateFunc {
 		auto &casts = config.GetCastFunctions();
 
 		casts.RegisterCastFunction(LogicalType::VARCHAR, LogicalType::TIMESTAMP_TZ, BindCastFromVarchar);
+		casts.RegisterCastFunction(LogicalType::VARCHAR, LogicalType::TIME_TZ, BindCastFromVarchar);
 	}
 };
 

package/src/duckdb/extension/icu/icu-table-range.cpp

@@ -90,6 +90,11 @@ struct ICUTableRange {
 
 		auto &inputs = input.inputs;
 		D_ASSERT(inputs.size() == 3);
+		for (const auto &value : inputs) {
+			if (value.IsNull()) {
+				throw BinderException("RANGE with NULL bounds is not supported");
+			}
+		}
 		result->start = inputs[0].GetValue<timestamp_t>();
 		result->end = inputs[1].GetValue<timestamp_t>();
 		result->increment = inputs[2].GetValue<interval_t>();

package/src/duckdb/extension/icu/icu-timezone.cpp

@@ -8,6 +8,7 @@
 #include "duckdb/parser/parsed_data/create_table_function_info.hpp"
 #include "include/icu-datefunc.hpp"
 #include "duckdb/transaction/meta_transaction.hpp"
+#include "duckdb/common/operator/cast_operators.hpp"
 
 namespace duckdb {
 
@@ -138,13 +139,23 @@ struct ICUFromNaiveTimestamp : public ICUDateFunc {
 		return GetTime(calendar, micros);
 	}
 
+	struct CastTimestampUsToUs {
+		template <class SRC, class DST>
+		static inline DST Operation(SRC input) {
+			// no-op
+			return input;
+		}
+	};
+
+	template <class OP>
 	static bool CastFromNaive(Vector &source, Vector &result, idx_t count, CastParameters &parameters) {
 		auto &cast_data = parameters.cast_data->Cast<CastData>();
 		auto &info = cast_data.info->Cast<BindData>();
 		CalendarPtr calendar(info.calendar->clone());
 
-		UnaryExecutor::Execute<timestamp_t, timestamp_t>(
-		    source, result, count, [&](timestamp_t input) { return Operation(calendar.get(), input); });
+		UnaryExecutor::Execute<timestamp_t, timestamp_t>(source, result, count, [&](timestamp_t input) {
+			return Operation(calendar.get(), OP::template Operation<timestamp_t, timestamp_t>(input));
+		});
 		return true;
 	}
 
@@ -154,8 +165,18 @@ struct ICUFromNaiveTimestamp : public ICUDateFunc {
 		}
 
 		auto cast_data = make_uniq<CastData>(make_uniq<BindData>(*input.context));
-
-		return BoundCastInfo(CastFromNaive, std::move(cast_data));
+		switch (source.id()) {
+		case LogicalTypeId::TIMESTAMP:
+			return BoundCastInfo(CastFromNaive<CastTimestampUsToUs>, std::move(cast_data));
+		case LogicalTypeId::TIMESTAMP_MS:
+			return BoundCastInfo(CastFromNaive<CastTimestampMsToUs>, std::move(cast_data));
+		case LogicalTypeId::TIMESTAMP_NS:
+			return BoundCastInfo(CastFromNaive<CastTimestampNsToUs>, std::move(cast_data));
+		case LogicalTypeId::TIMESTAMP_SEC:
+			return BoundCastInfo(CastFromNaive<CastTimestampSecToUs>, std::move(cast_data));
+		default:
+			throw InternalException("Type %s not handled in BindCastFromNaive", LogicalTypeIdToString(source.id()));
+		}
 	}
 
 	static void AddCasts(DatabaseInstance &db) {
@@ -163,6 +184,9 @@ struct ICUFromNaiveTimestamp : public ICUDateFunc {
 		auto &casts = config.GetCastFunctions();
 
 		casts.RegisterCastFunction(LogicalType::TIMESTAMP, LogicalType::TIMESTAMP_TZ, BindCastFromNaive);
+		casts.RegisterCastFunction(LogicalType::TIMESTAMP_MS, LogicalType::TIMESTAMP_TZ, BindCastFromNaive);
+		casts.RegisterCastFunction(LogicalType::TIMESTAMP_NS, LogicalType::TIMESTAMP_TZ, BindCastFromNaive);
+		casts.RegisterCastFunction(LogicalType::TIMESTAMP_S, LogicalType::TIMESTAMP_TZ, BindCastFromNaive);
 	}
 };
 

package/src/duckdb/extension/json/buffered_json_reader.cpp

@@ -304,11 +304,11 @@ idx_t BufferedJSONReader::GetLineNumber(idx_t buf_index, idx_t line_or_object_in
 					line += buffer_line_or_object_counts[b_idx];
 				}
 			}
-		}
-		if (can_throw) {
-			thrown = true;
-			// SQL uses 1-based indexing so I guess we will do that in our exception here as well
-			return line + 1;
+			if (can_throw) {
+				thrown = true;
+				// SQL uses 1-based indexing so I guess we will do that in our exception here as well
+				return line + 1;
+			}
 		}
 		TaskScheduler::YieldThread();
 	}
@@ -331,6 +331,7 @@ void BufferedJSONReader::ThrowTransformError(idx_t buf_index, idx_t line_or_obje
 }
 
 double BufferedJSONReader::GetProgress() const {
+	lock_guard<mutex> guard(lock);
 	if (HasFileHandle()) {
 		return 100.0 - 100.0 * double(file_handle->Remaining()) / double(file_handle->FileSize());
 	} else {

package/src/duckdb/extension/json/include/buffered_json_reader.hpp

@@ -90,9 +90,9 @@ private:
 
 	//! Read properties
 	idx_t read_position;
-	idx_t requested_reads;
+	atomic<idx_t> requested_reads;
 	atomic<idx_t> actual_reads;
-	bool last_read_requested;
+	atomic<bool> last_read_requested;
 
 	//! Cached buffers for resetting when reading stream
 	vector<AllocatedData> cached_buffers;
@@ -161,7 +161,7 @@ private:
 	bool thrown;
 
 public:
-	mutex lock;
+	mutable mutex lock;
 	MultiFileReaderData reader_data;
 };
 

package/src/duckdb/extension/json/include/json_transform.hpp

@@ -40,6 +40,8 @@ public:
 	string error_message;
 	//! Index of the object where the error occurred
 	idx_t object_index = DConstants::INVALID_INDEX;
+	//! Cast parameters
+	CastParameters parameters;
 
 public:
 	void Serialize(Serializer &serializer) const;

package/src/duckdb/extension/json/json_functions/json_create.cpp

@@ -61,6 +61,9 @@ static LogicalType GetJSONType(StructNames &const_struct_names, const LogicalTyp
 	// The nested types need to conform as well
 	case LogicalTypeId::LIST:
 		return LogicalType::LIST(GetJSONType(const_struct_names, ListType::GetChildType(type)));
+	case LogicalTypeId::ARRAY:
+		return LogicalType::ARRAY(GetJSONType(const_struct_names, ArrayType::GetChildType(type)),
+		                          ArrayType::GetSize(type));
 	// Struct and MAP are treated as JSON values
 	case LogicalTypeId::STRUCT: {
 		child_list_t<LogicalType> child_types;
@@ -435,6 +438,9 @@ static void CreateValuesList(const StructNames &names, yyjson_mut_doc *doc, yyjs
 
 static void CreateValuesArray(const StructNames &names, yyjson_mut_doc *doc, yyjson_mut_val *vals[], Vector &value_v,
                               idx_t count) {
+
+	value_v.Flatten(count);
+
 	// Initialize array for the nested values
 	auto &child_v = ArrayVector::GetEntry(value_v);
 	auto array_size = ArrayType::GetSize(value_v.GetType());

package/src/duckdb/extension/json/json_functions/json_transform.cpp

@@ -13,13 +13,13 @@
 
 namespace duckdb {
 
-JSONTransformOptions::JSONTransformOptions() {
+JSONTransformOptions::JSONTransformOptions() : parameters(false, &error_message) {
 }
 
 JSONTransformOptions::JSONTransformOptions(bool strict_cast_p, bool error_duplicate_key_p, bool error_missing_key_p,
                                            bool error_unkown_key_p)
     : strict_cast(strict_cast_p), error_duplicate_key(error_duplicate_key_p), error_missing_key(error_missing_key_p),
-      error_unknown_key(error_unkown_key_p) {
+      error_unknown_key(error_unkown_key_p), parameters(false, &error_message) {
 }
 
 //! Forward declaration for recursion
@@ -135,7 +135,7 @@ static inline bool GetValueDecimal(yyjson_val *val, T &result, uint8_t w, uint8_
 	bool success;
 	switch (unsafe_yyjson_get_tag(val)) {
 	case YYJSON_TYPE_STR | YYJSON_SUBTYPE_NONE:
-		success = OP::template Operation<string_t, T>(GetString(val), result, &options.error_message, w, s);
+		success = OP::template Operation<string_t, T>(GetString(val), result, options.parameters, w, s);
 		break;
 	case YYJSON_TYPE_ARR | YYJSON_SUBTYPE_NONE:
 	case YYJSON_TYPE_OBJ | YYJSON_SUBTYPE_NONE:
@@ -143,17 +143,16 @@ static inline bool GetValueDecimal(yyjson_val *val, T &result, uint8_t w, uint8_
 		break;
 	case YYJSON_TYPE_BOOL | YYJSON_SUBTYPE_TRUE:
 	case YYJSON_TYPE_BOOL | YYJSON_SUBTYPE_FALSE:
-		success = OP::template Operation<bool, T>(unsafe_yyjson_get_bool(val), result, &options.error_message, w, s);
+		success = OP::template Operation<bool, T>(unsafe_yyjson_get_bool(val), result, options.parameters, w, s);
 		break;
 	case YYJSON_TYPE_NUM | YYJSON_SUBTYPE_UINT:
-		success =
-		    OP::template Operation<uint64_t, T>(unsafe_yyjson_get_uint(val), result, &options.error_message, w, s);
+		success = OP::template Operation<uint64_t, T>(unsafe_yyjson_get_uint(val), result, options.parameters, w, s);
 		break;
 	case YYJSON_TYPE_NUM | YYJSON_SUBTYPE_SINT:
-		success = OP::template Operation<int64_t, T>(unsafe_yyjson_get_sint(val), result, &options.error_message, w, s);
+		success = OP::template Operation<int64_t, T>(unsafe_yyjson_get_sint(val), result, options.parameters, w, s);
 		break;
 	case YYJSON_TYPE_NUM | YYJSON_SUBTYPE_REAL:
-		success = OP::template Operation<double, T>(unsafe_yyjson_get_real(val), result, &options.error_message, w, s);
+		success = OP::template Operation<double, T>(unsafe_yyjson_get_real(val), result, options.parameters, w, s);
 		break;
 	default:
 		throw InternalException("Unknown yyjson tag in GetValueString");
@@ -982,7 +981,7 @@ static bool JSONToAnyCast(Vector &source, Vector &result, idx_t count, CastParam
 
 	auto success = TransformFunctionInternal(source, count, result, alc, options);
 	if (!success) {
-		HandleCastError::AssignError(options.error_message, parameters.error_message);
+		HandleCastError::AssignError(options.error_message, parameters);
 	}
 	return success;
 }

package/src/duckdb/extension/json/json_functions.cpp

@@ -224,8 +224,7 @@ static bool CastVarcharToJSON(Vector &source, Vector &result, idx_t count, CastP
 		    if (!doc) {
 			    mask.SetInvalid(idx);
 			    if (success) {
-				    HandleCastError::AssignError(JSONCommon::FormatParseError(data, length, error),
-				                                 parameters.error_message);
+				    HandleCastError::AssignError(JSONCommon::FormatParseError(data, length, error), parameters);
 				    success = false;
 			    }
 		    }

package/src/duckdb/extension/json/json_scan.cpp

@@ -569,6 +569,7 @@ bool JSONScanLocalState::ReadNextBuffer(JSONScanGlobalState &gstate) {
 			if (file_done) {
 				lock_guard<mutex> guard(gstate.lock);
 				TryIncrementFileIndex(gstate);
+				lock_guard<mutex> reader_guard(current_reader->lock);
 				current_reader->GetFileHandle().Close();
 			}
 

package/src/duckdb/extension/parquet/column_reader.cpp

@@ -1004,7 +1004,23 @@ idx_t CastColumnReader::Read(uint64_t num_values, parquet_filter_t &filter, data
 			}
 		}
 	}
-	VectorOperations::DefaultCast(intermediate_vector, result, amount);
+	string error_message;
+	bool all_succeeded = VectorOperations::DefaultTryCast(intermediate_vector, result, amount, &error_message);
+	if (!all_succeeded) {
+		string extended_error;
+		extended_error =
+		    StringUtil::Format("In file \"%s\" the column \"%s\" has type %s, but we are trying to read it as type %s.",
+		                       reader.file_name, schema.name, intermediate_vector.GetType(), result.GetType());
+		extended_error += "\nThis can happen when reading multiple Parquet files. The schema information is taken from "
+		                  "the first Parquet file by default. Possible solutions:\n";
+		extended_error += "* Enable the union_by_name=True option to combine the schema of all Parquet files "
+		                  "(duckdb.org/docs/data/multiple_files/combining_schemas)\n";
+		extended_error += "* Use a COPY statement to automatically derive types from an existing table.";
+		throw ConversionException(
+		    "In Parquet reader of file \"%s\": failed to cast column \"%s\" from type %s to %s: %s\n\n%s",
+		    reader.file_name, schema.name, intermediate_vector.GetType(), result.GetType(), error_message,
+		    extended_error);
+	}
 	return amount;
 }