duckdb 0.9.1-dev97.0 → 0.9.2-dev10.0

package/src/duckdb/src/parser/transform/statement/transform_create_sequence.cpp

@@ -17,6 +17,8 @@ unique_ptr<CreateStatement> Transformer::TransformCreateSequence(duckdb_libpgque
 	info->name = qname.name;
 
 	if (stmt.options) {
+		int64_t default_start_value = info->start_value;
+		bool has_start_value = false;
 		unordered_set<SequenceInfo, EnumClassHash> used;
 		duckdb_libpgquery::PGListCell *cell = nullptr;
 		for_each_cell(cell, stmt.options->head) {
@@ -51,10 +53,10 @@ unique_ptr<CreateStatement> Transformer::TransformCreateSequence(duckdb_libpgque
 					throw ParserException("Increment must not be zero");
 				}
 				if (info->increment < 0) {
-					info->start_value = info->max_value = -1;
+					default_start_value = info->max_value = -1;
 					info->min_value = NumericLimits<int64_t>::Minimum();
 				} else {
-					info->start_value = info->min_value = 1;
+					default_start_value = info->min_value = 1;
 					info->max_value = NumericLimits<int64_t>::Maximum();
 				}
 			} else if (opt_name == "minvalue") {
@@ -68,7 +70,7 @@ unique_ptr<CreateStatement> Transformer::TransformCreateSequence(duckdb_libpgque
 
 				info->min_value = opt_value;
 				if (info->increment > 0) {
-					info->start_value = info->min_value;
+					default_start_value = info->min_value;
 				}
 			} else if (opt_name == "maxvalue") {
 				if (used.find(SequenceInfo::SEQ_MAX) != used.end()) {
@@ -81,7 +83,7 @@ unique_ptr<CreateStatement> Transformer::TransformCreateSequence(duckdb_libpgque
 
 				info->max_value = opt_value;
 				if (info->increment < 0) {
-					info->start_value = info->max_value;
+					default_start_value = info->max_value;
 				}
 			} else if (opt_name == "start") {
 				if (used.find(SequenceInfo::SEQ_START) != used.end()) {
@@ -91,7 +93,7 @@ unique_ptr<CreateStatement> Transformer::TransformCreateSequence(duckdb_libpgque
 				if (nodef) {
 					continue;
 				}
-
+				has_start_value = true;
 				info->start_value = opt_value;
 			} else if (opt_name == "cycle") {
 				if (used.find(SequenceInfo::SEQ_CYCLE) != used.end()) {
@@ -107,6 +109,9 @@ unique_ptr<CreateStatement> Transformer::TransformCreateSequence(duckdb_libpgque
 				throw ParserException("Unrecognized option \"%s\" for CREATE SEQUENCE", opt_name);
 			}
 		}
+		if (!has_start_value) {
+			info->start_value = default_start_value;
+		}
 	}
 	info->temporary = !stmt.sequence->relpersistence;
 	info->on_conflict = TransformOnConflict(stmt.onconflict);

package/src/duckdb/src/planner/binder/expression/bind_between_expression.cpp

@@ -34,13 +34,11 @@ BindResult ExpressionBinder::BindExpression(BetweenExpression &expr, idx_t depth
 	input = BoundCastExpression::AddCastToType(context, std::move(input), input_type);
 	lower = BoundCastExpression::AddCastToType(context, std::move(lower), input_type);
 	upper = BoundCastExpression::AddCastToType(context, std::move(upper), input_type);
-	if (input_type.id() == LogicalTypeId::VARCHAR) {
-		// handle collation
-		auto collation = StringType::GetCollation(input_type);
-		input = PushCollation(context, std::move(input), collation, false);
-		lower = PushCollation(context, std::move(lower), collation, false);
-		upper = PushCollation(context, std::move(upper), collation, false);
-	}
+	// handle collation
+	PushCollation(context, input, input_type, false);
+	PushCollation(context, lower, input_type, false);
+	PushCollation(context, upper, input_type, false);
+
 	if (!input->HasSideEffects() && !input->HasParameter() && !input->HasSubquery()) {
 		// the expression does not have side effects and can be copied: create two comparisons
 		// the reason we do this is that individual comparisons are easier to handle in optimizers

package/src/duckdb/src/planner/binder/expression/bind_collate_expression.cpp

@@ -18,8 +18,10 @@ BindResult ExpressionBinder::BindExpression(CollateExpression &expr, idx_t depth
 		throw BinderException("collations are only supported for type varchar");
 	}
 	// Validate the collation, but don't use it
-	PushCollation(context, child->Copy(), expr.collation, false);
-	child->return_type = LogicalType::VARCHAR_COLLATION(expr.collation);
+	auto child_copy = child->Copy();
+	auto collation_type = LogicalType::VARCHAR_COLLATION(expr.collation);
+	PushCollation(context, child_copy, collation_type, false);
+	child->return_type = collation_type;
 	return BindResult(std::move(child));
 }
 

package/src/duckdb/src/planner/binder/expression/bind_comparison_expression.cpp

@@ -18,20 +18,25 @@
 
 namespace duckdb {
 
-unique_ptr<Expression> ExpressionBinder::PushCollation(ClientContext &context, unique_ptr<Expression> source,
-                                                       const string &collation_p, bool equality_only) {
+bool ExpressionBinder::PushCollation(ClientContext &context, unique_ptr<Expression> &source,
+                                     const LogicalType &sql_type, bool equality_only) {
+	if (sql_type.id() != LogicalTypeId::VARCHAR) {
+		// only VARCHAR columns require collation
+		return false;
+	}
 	// replace default collation with system collation
+	auto str_collation = StringType::GetCollation(sql_type);
 	string collation;
-	if (collation_p.empty()) {
+	if (str_collation.empty()) {
 		collation = DBConfig::GetConfig(context).options.collation;
 	} else {
-		collation = collation_p;
+		collation = str_collation;
 	}
 	collation = StringUtil::Lower(collation);
 	// bind the collation
 	if (collation.empty() || collation == "binary" || collation == "c" || collation == "posix") {
-		// binary collation: just skip
-		return source;
+		// no collation or binary collation: skip
+		return false;
 	}
 	auto &catalog = Catalog::GetSystemCatalog(context);
 	auto splits = StringUtil::Split(StringUtil::Lower(collation), ".");
@@ -60,11 +65,12 @@ unique_ptr<Expression> ExpressionBinder::PushCollation(ClientContext &context, u
 		auto function = function_binder.BindScalarFunction(collation_entry.function, std::move(children));
 		source = std::move(function);
 	}
-	return source;
+	return true;
 }
 
 void ExpressionBinder::TestCollation(ClientContext &context, const string &collation) {
-	PushCollation(context, make_uniq<BoundConstantExpression>(Value("")), collation);
+	auto expr = make_uniq_base<Expression, BoundConstantExpression>(Value(""));
+	PushCollation(context, expr, LogicalType::VARCHAR_COLLATION(collation));
 }
 
 LogicalType BoundComparisonExpression::BindComparison(LogicalType left_type, LogicalType right_type) {
@@ -134,12 +140,9 @@ BindResult ExpressionBinder::BindExpression(ComparisonExpression &expr, idx_t de
 	right = BoundCastExpression::AddCastToType(context, std::move(right), input_type,
 	                                           input_type.id() == LogicalTypeId::ENUM);
 
-	if (input_type.id() == LogicalTypeId::VARCHAR) {
-		// handle collation
-		auto collation = StringType::GetCollation(input_type);
-		left = PushCollation(context, std::move(left), collation, expr.type == ExpressionType::COMPARE_EQUAL);
-		right = PushCollation(context, std::move(right), collation, expr.type == ExpressionType::COMPARE_EQUAL);
-	}
+	PushCollation(context, left, input_type, expr.type == ExpressionType::COMPARE_EQUAL);
+	PushCollation(context, right, input_type, expr.type == ExpressionType::COMPARE_EQUAL);
+
 	// now create the bound comparison expression
 	return BindResult(make_uniq<BoundComparisonExpression>(expr.type, std::move(left), std::move(right)));
 }

package/src/duckdb/src/planner/binder/query_node/bind_select_node.cpp

@@ -222,10 +222,7 @@ void Binder::BindModifierTypes(BoundQueryNode &result, const vector<LogicalType>
 			for (auto &target_distinct : distinct.target_distincts) {
 				auto &bound_colref = target_distinct->Cast<BoundColumnRefExpression>();
 				const auto &sql_type = sql_types[bound_colref.binding.column_index];
-				if (sql_type.id() == LogicalTypeId::VARCHAR) {
-					target_distinct = ExpressionBinder::PushCollation(context, std::move(target_distinct),
-					                                                  StringType::GetCollation(sql_type), true);
-				}
+				ExpressionBinder::PushCollation(context, target_distinct, sql_type, true);
 			}
 			break;
 		}
@@ -253,10 +250,7 @@ void Binder::BindModifierTypes(BoundQueryNode &result, const vector<LogicalType>
 				D_ASSERT(bound_colref.binding.column_index < sql_types.size());
 				const auto &sql_type = sql_types[bound_colref.binding.column_index];
 				bound_colref.return_type = sql_types[bound_colref.binding.column_index];
-				if (sql_type.id() == LogicalTypeId::VARCHAR) {
-					order_node.expression = ExpressionBinder::PushCollation(context, std::move(order_node.expression),
-					                                                        StringType::GetCollation(sql_type));
-				}
+				ExpressionBinder::PushCollation(context, order_node.expression, sql_type);
 			}
 			break;
 		}
@@ -389,9 +383,8 @@ unique_ptr<BoundQueryNode> Binder::BindSelectNode(SelectNode &statement, unique_
 			bool contains_subquery = bound_expr_ref.HasSubquery();
 
 			// push a potential collation, if necessary
-			auto collated_expr = ExpressionBinder::PushCollation(context, std::move(bound_expr),
-			                                                     StringType::GetCollation(group_type), true);
-			if (!contains_subquery && !collated_expr->Equals(bound_expr_ref)) {
+			bool requires_collation = ExpressionBinder::PushCollation(context, bound_expr, group_type, true);
+			if (!contains_subquery && requires_collation) {
 				// if there is a collation on a group x, we should group by the collated expr,
 				// but also push a first(x) aggregate in case x is selected (uncollated)
 				info.collated_groups[i] = result->aggregates.size();
@@ -405,7 +398,7 @@ unique_ptr<BoundQueryNode> Binder::BindSelectNode(SelectNode &statement, unique_
 				auto function = function_binder.BindAggregateFunction(first_fun, std::move(first_children));
 				result->aggregates.push_back(std::move(function));
 			}
-			result->groups.group_expressions.push_back(std::move(collated_expr));
+			result->groups.group_expressions.push_back(std::move(bound_expr));
 
 			// in the unbound expression we DO bind the table names of any ColumnRefs
 			// we do this to make sure that "table.a" and "a" are treated the same

package/src/duckdb/src/planner/binder/statement/bind_create.cpp

@@ -68,6 +68,16 @@ void Binder::BindSchemaOrCatalog(string &catalog, string &schema) {
 	BindSchemaOrCatalog(context, catalog, schema);
 }
 
+const string Binder::BindCatalog(string &catalog) {
+	auto &db_manager = DatabaseManager::Get(context);
+	optional_ptr<AttachedDatabase> database = db_manager.GetDatabase(context, catalog);
+	if (database) {
+		return db_manager.GetDatabase(context, catalog).get()->GetName();
+	} else {
+		return db_manager.GetDefaultDatabase(context);
+	}
+}
+
 SchemaCatalogEntry &Binder::BindSchema(CreateInfo &info) {
 	BindSchemaOrCatalog(info.catalog, info.schema);
 	if (IsInvalidCatalog(info.catalog) && info.temporary) {
@@ -456,9 +466,13 @@ BoundStatement Binder::Bind(CreateStatement &stmt) {
 
 	auto catalog_type = stmt.info->type;
 	switch (catalog_type) {
-	case CatalogType::SCHEMA_ENTRY:
+	case CatalogType::SCHEMA_ENTRY: {
+		auto &base = stmt.info->Cast<CreateInfo>();
+		auto catalog = BindCatalog(base.catalog);
+		properties.modified_databases.insert(catalog);
 		result.plan = make_uniq<LogicalCreate>(LogicalOperatorType::LOGICAL_CREATE_SCHEMA, std::move(stmt.info));
 		break;
+	}
 	case CatalogType::VIEW_ENTRY: {
 		auto &base = stmt.info->Cast<CreateViewInfo>();
 		// bind the schema

package/src/duckdb/src/planner/bound_parameter_map.cpp

@@ -43,19 +43,30 @@ shared_ptr<BoundParameterData> BoundParameterMap::CreateOrGetData(const string &
 }
 
 unique_ptr<BoundParameterExpression> BoundParameterMap::BindParameterExpression(ParameterExpression &expr) {
-	auto &identifier = expr.identifier;
-	auto return_type = GetReturnType(identifier);
 
+	auto &identifier = expr.identifier;
 	D_ASSERT(!parameter_data.count(identifier));
 
 	// No value has been supplied yet,
-	// We return a shared pointer to an object that will get populated wtih a Value later
-	// When the BoundParameterExpression get executed, this will be used to get the corresponding value
+	// We return a shared pointer to an object that will get populated with a Value later
+	// When the BoundParameterExpression gets executed, this will be used to get the corresponding value
 	auto param_data = CreateOrGetData(identifier);
 	auto bound_expr = make_uniq<BoundParameterExpression>(identifier);
+
 	bound_expr->parameter_data = param_data;
-	bound_expr->return_type = return_type;
 	bound_expr->alias = expr.alias;
+
+	auto param_type = param_data->return_type;
+	auto identifier_type = GetReturnType(identifier);
+
+	// we found a type for this bound parameter, but now we found another occurrence with the same identifier,
+	// a CAST around this consecutive occurrence might swallow the unknown type of this consecutive occurrence,
+	// then, if we do not rebind, we potentially have unknown data types during execution
+	if (identifier_type == LogicalType::UNKNOWN && param_type != LogicalType::UNKNOWN) {
+		rebind = true;
+	}
+
+	bound_expr->return_type = identifier_type;
 	return bound_expr;
 }
 

package/src/duckdb/src/planner/expression_binder/base_select_binder.cpp

@@ -98,11 +98,8 @@ BindResult BaseSelectBinder::BindColumnRef(unique_ptr<ParsedExpression> &expr_pt
 				                      " This is not yet supported.",
 				                      colref.column_names[0]);
 			}
-			auto result = BindResult(node.select_list[index]->Copy());
-			if (result.expression->type == ExpressionType::BOUND_COLUMN_REF) {
-				auto &result_expr = result.expression->Cast<BoundColumnRefExpression>();
-				result_expr.depth = depth;
-			}
+			auto copied_expression = node.original_expressions[index]->Copy();
+			result = BindExpression(copied_expression, depth, false);
 			return result;
 		}
 	}

package/src/duckdb/src/planner/planner.cpp

@@ -76,7 +76,7 @@ void Planner::CreatePlan(SQLStatement &statement) {
 	}
 	this->properties = binder->properties;
 	this->properties.parameter_count = parameter_count;
-	properties.bound_all_parameters = parameters_resolved;
+	properties.bound_all_parameters = !bound_parameters.rebind && parameters_resolved;
 
 	Planner::VerifyPlan(context, plan, bound_parameters.GetParametersPtr());
 

package/src/duckdb/src/transaction/duck_transaction_manager.cpp

@@ -252,6 +252,7 @@ void DuckTransactionManager::RollbackTransaction(Transaction *transaction_p) {
 }
 
 void DuckTransactionManager::RemoveTransaction(DuckTransaction &transaction) noexcept {
+	bool changes_made = transaction.ChangesMade();
 	// remove the transaction from the list of active transactions
 	idx_t t_index = active_transactions.size();
 	// check for the lowest and highest start time in the list of transactions
@@ -275,15 +276,18 @@ void DuckTransactionManager::RemoveTransaction(DuckTransaction &transaction) noe
 	D_ASSERT(t_index != active_transactions.size());
 	auto current_transaction = std::move(active_transactions[t_index]);
 	auto current_query = DatabaseManager::Get(db).ActiveQueryNumber();
-	if (transaction.commit_id != 0) {
-		// the transaction was committed, add it to the list of recently
-		// committed transactions
-		recently_committed_transactions.push_back(std::move(current_transaction));
-	} else {
-		// the transaction was aborted, but we might still need its information
-		// add it to the set of transactions awaiting GC
-		current_transaction->highest_active_query = current_query;
-		old_transactions.push_back(std::move(current_transaction));
+	if (changes_made) {
+		// if the transaction made any changes we need to keep it around
+		if (transaction.commit_id != 0) {
+			// the transaction was committed, add it to the list of recently
+			// committed transactions
+			recently_committed_transactions.push_back(std::move(current_transaction));
+		} else {
+			// the transaction was aborted, but we might still need its information
+			// add it to the set of transactions awaiting GC
+			current_transaction->highest_active_query = current_query;
+			old_transactions.push_back(std::move(current_transaction));
+		}
 	}
 	// remove the transaction from the set of currently active transactions
 	active_transactions.erase(active_transactions.begin() + t_index);

package/src/duckdb/third_party/parquet/parquet_types.h

@@ -92,7 +92,8 @@ struct Encoding {
     DELTA_BINARY_PACKED = 5,
     DELTA_LENGTH_BYTE_ARRAY = 6,
     DELTA_BYTE_ARRAY = 7,
-    RLE_DICTIONARY = 8
+    RLE_DICTIONARY = 8,
+    BYTE_STREAM_SPLIT = 9,
   };
 };
 

package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp

@@ -348,17 +348,17 @@
 
 #include "extension/icu/third_party/icu/i18n/wintzimpl.cpp"
 
-#include "extension/icu/third_party/icu/i18n/double-conversion-strtod.cpp"
+#include "extension/icu/third_party/icu/i18n/double-conversion-fast-dtoa.cpp"
 
-#include "extension/icu/third_party/icu/i18n/double-conversion-double-to-string.cpp"
+#include "extension/icu/third_party/icu/i18n/double-conversion-cached-powers.cpp"
 
-#include "extension/icu/third_party/icu/i18n/double-conversion-fast-dtoa.cpp"
+#include "extension/icu/third_party/icu/i18n/double-conversion-strtod.cpp"
 
-#include "extension/icu/third_party/icu/i18n/double-conversion-string-to-double.cpp"
+#include "extension/icu/third_party/icu/i18n/double-conversion-double-to-string.cpp"
 
 #include "extension/icu/third_party/icu/i18n/double-conversion-bignum-dtoa.cpp"
 
 #include "extension/icu/third_party/icu/i18n/double-conversion-bignum.cpp"
 
-#include "extension/icu/third_party/icu/i18n/double-conversion-cached-powers.cpp"
+#include "extension/icu/third_party/icu/i18n/double-conversion-string-to-double.cpp"
 

package/src/duckdb/ub_src_optimizer_pushdown.cpp

@@ -2,6 +2,8 @@
 
 #include "src/optimizer/pushdown/pushdown_cross_product.cpp"
 
+#include "src/optimizer/pushdown/pushdown_distinct.cpp"
+
 #include "src/optimizer/pushdown/pushdown_filter.cpp"
 
 #include "src/optimizer/pushdown/pushdown_get.cpp"

package/src/duckdb_node.cpp

@@ -12,15 +12,36 @@ NodeDuckDB::NodeDuckDB(Napi::Env env, Napi::Object exports) {
 	statement_constructor = node_duckdb::Statement::Init(env, exports);
 	query_result_constructor = node_duckdb::QueryResult::Init(env, exports);
 
-	exports.DefineProperties({
-	    DEFINE_CONSTANT_INTEGER(exports, node_duckdb::Database::DUCKDB_NODEJS_ERROR, ERROR) DEFINE_CONSTANT_INTEGER(
+	auto token_type_enum = Napi::Object::New(env);
+
+	token_type_enum.Set("IDENTIFIER", 0);
+	token_type_enum.Set("NUMERIC_CONSTANT", 1);
+	token_type_enum.Set("STRING_CONSTANT", 2);
+	token_type_enum.Set("OPERATOR", 3);
+	token_type_enum.Set("KEYWORD", 4);
+	token_type_enum.Set("COMMENT", 5);
+
+	// TypeScript enums expose an inverse mapping.
+	token_type_enum.Set((uint32_t)0, "IDENTIFIER");
+	token_type_enum.Set((uint32_t)1, "NUMERIC_CONSTANT");
+	token_type_enum.Set((uint32_t)2, "STRING_CONSTANT");
+	token_type_enum.Set((uint32_t)3, "OPERATOR");
+	token_type_enum.Set((uint32_t)4, "KEYWORD");
+	token_type_enum.Set((uint32_t)5, "COMMENT");
+
+	token_type_enum_ref = Napi::ObjectReference::New(token_type_enum);
+
+	exports.DefineProperties(
+	    {DEFINE_CONSTANT_INTEGER(exports, node_duckdb::Database::DUCKDB_NODEJS_ERROR, ERROR) DEFINE_CONSTANT_INTEGER(
 	        exports, node_duckdb::Database::DUCKDB_NODEJS_READONLY, OPEN_READONLY) // same as SQLite
-	    DEFINE_CONSTANT_INTEGER(exports, 0, OPEN_READWRITE)                        // ignored
-	    DEFINE_CONSTANT_INTEGER(exports, 0, OPEN_CREATE)                           // ignored
-	    DEFINE_CONSTANT_INTEGER(exports, 0, OPEN_FULLMUTEX)                        // ignored
-	    DEFINE_CONSTANT_INTEGER(exports, 0, OPEN_SHAREDCACHE)                      // ignored
-	    DEFINE_CONSTANT_INTEGER(exports, 0, OPEN_PRIVATECACHE)                     // ignored
-	});
+	     DEFINE_CONSTANT_INTEGER(exports, 0, OPEN_READWRITE)                       // ignored
+	     DEFINE_CONSTANT_INTEGER(exports, 0, OPEN_CREATE)                          // ignored
+	     DEFINE_CONSTANT_INTEGER(exports, 0, OPEN_FULLMUTEX)                       // ignored
+	     DEFINE_CONSTANT_INTEGER(exports, 0, OPEN_SHAREDCACHE)                     // ignored
+	     DEFINE_CONSTANT_INTEGER(exports, 0, OPEN_PRIVATECACHE)                    // ignored
+
+	     Napi::PropertyDescriptor::Value("TokenType", token_type_enum,
+	                                     static_cast<napi_property_attributes>(napi_enumerable | napi_configurable))});
 }
 
 NODE_API_ADDON(NodeDuckDB);

package/src/duckdb_node.hpp

@@ -23,6 +23,7 @@ public:
 	Napi::FunctionReference connection_constructor;
 	Napi::FunctionReference statement_constructor;
 	Napi::FunctionReference query_result_constructor;
+	Napi::ObjectReference token_type_enum_ref;
 };
 
 namespace node_duckdb {
@@ -109,6 +110,7 @@ public:
 	Napi::Value Interrupt(const Napi::CallbackInfo &info);
 	Napi::Value Close(const Napi::CallbackInfo &info);
 	Napi::Value RegisterReplacementScan(const Napi::CallbackInfo &info);
+	Napi::Value Tokenize(const Napi::CallbackInfo &info);
 
 public:
 	constexpr static int DUCKDB_NODEJS_ERROR = -1;

package/src/statement.cpp

@@ -187,6 +187,15 @@ static Napi::Value convert_col_val(Napi::Env &env, duckdb::Value dval, duckdb::L
 		const auto scale = duckdb::Interval::SECS_PER_DAY * duckdb::Interval::MSECS_PER_SEC;
 		value = Napi::Date::New(env, double(dval.GetValue<int32_t>() * scale));
 	} break;
+	case duckdb::LogicalTypeId::TIMESTAMP_NS: {
+		value = Napi::Date::New(env, double(dval.GetValue<int64_t>() / (duckdb::Interval::MICROS_PER_MSEC * 1000)));
+	} break;
+	case duckdb::LogicalTypeId::TIMESTAMP_MS: {
+		value = Napi::Date::New(env, double(dval.GetValue<int64_t>()));
+	} break;
+	case duckdb::LogicalTypeId::TIMESTAMP_SEC: {
+		value = Napi::Date::New(env, double(dval.GetValue<int64_t>() * duckdb::Interval::MSECS_PER_SEC));
+	} break;
 	case duckdb::LogicalTypeId::TIMESTAMP:
 	case duckdb::LogicalTypeId::TIMESTAMP_TZ: {
 		value = Napi::Date::New(env, double(dval.GetValue<int64_t>() / duckdb::Interval::MICROS_PER_MSEC));
@@ -377,10 +386,10 @@ struct RunPreparedTask : public Task {
 					// query results, the string data is owned by the QueryResult
 					auto result_ref_ptr = new std::shared_ptr<duckdb::QueryResult>(result_ptr);
 
-					auto array_buffer =
-					    Napi::ArrayBuffer::New(env, (void *)blob.GetData(), blob.GetSize(), deleter, result_ref_ptr);
+					auto array_buffer = Napi::Buffer<char>::NewOrCopy(env, (char *)blob.GetData(), blob.GetSize(),
+					                                                  deleter, result_ref_ptr);
 
-					auto typed_array = Napi::Uint8Array::New(env, blob.GetSize(), array_buffer, 0);
+					auto typed_array = Napi::TypedArrayOf<char>(env, array_buffer);
 
 					// TODO we should handle this in duckdb probably
 					if (is_header) {
@@ -708,7 +717,7 @@ struct GetNextArrowIpcTask : public Task {
 			delete static_cast<unique_ptr<duckdb::DataChunk> *>(hint);
 		};
 		auto array_buffer =
-		    Napi::ArrayBuffer::New(env, (void *)blob.GetData(), blob.GetSize(), deleter, data_chunk_ptr);
+		    Napi::Buffer<char>::NewOrCopy(env, (char *)blob.GetData(), blob.GetSize(), deleter, data_chunk_ptr);
 
 		deferred.Resolve(array_buffer);
 	}

package/test/arrow.test.ts

@@ -9,7 +9,9 @@ describe('arrow IPC API fails neatly when extension not loaded', function() {
     let conn;
     before((done) => {
         db = new duckdb.Database(':memory:', {"allow_unsigned_extensions": "true"}, () => {
-            done();
+            db.all('SET autoload_known_extensions=false;', () => {
+                done();
+            });
         });
     });
 

package/test/parquet.test.ts

@@ -8,7 +8,7 @@ describe('can query parquet', function() {
     });
 
     it('should be able to read parquet files', function(done) {
-        db.run("select * from parquet_scan('../../data/parquet-testing/userdata1.parquet')", done);
+        db.run("select * from parquet_scan('test/userdata1.parquet')", done);
     });
 
 });

package/test/test_all_types.test.ts

@@ -22,10 +22,12 @@ function timedelta(obj: { days: number; micros: number; months: number }) {
 const replacement_values: Record<string, string> = {
   timestamp:
     "'1990-01-01 00:00:00'::TIMESTAMP, '9999-12-31 23:59:59'::TIMESTAMP, NULL::TIMESTAMP",
-  // TODO: fix these, they are currently being returned as strings
-  //   timestamp_s: "'1990-01-01 00:00:00'::TIMESTAMP_S",
-  //   timestamp_ns: "'1990-01-01 00:00:00'::TIMESTAMP_NS",
-  //   timestamp_ms: "'1990-01-01 00:00:00'::TIMESTAMP_MS",
+  timestamp_s:
+    "'1990-01-01 00:00:00'::TIMESTAMP_S, '9999-12-31 23:59:59'::TIMESTAMP_S, NULL::TIMESTAMP_S",
+  // note: timestamp_ns does not support extreme values
+  timestamp_ns: "'1990-01-01 00:00:00'::TIMESTAMP_NS,  NULL::TIMESTAMP_NS",
+  timestamp_ms:
+    "'1990-01-01 00:00:00'::TIMESTAMP_MS,  '9999-12-31 23:59:59'::TIMESTAMP_MS, NULL::TIMESTAMP_MS",
   timestamp_tz:
     "'1990-01-01 00:00:00Z'::TIMESTAMPTZ, '9999-12-31 23:59:59.999999Z'::TIMESTAMPTZ, NULL::TIMESTAMPTZ",
   date: "'1990-01-01'::DATE, '9999-12-31'::DATE, NULL::DATE",
@@ -157,7 +159,7 @@ const correct_answer_map: Record<string, any[]> = {
     null,
   ],
   map: ["{}", "{key1=🦆🦆🦆🦆🦆🦆, key2=goose}", null],
-  union: ['Frank', '5', null],
+  union: ["Frank", "5", null],
 
   time_tz: ["00:00:00-1559", "23:59:59.999999+1559", null],
   interval: [
@@ -176,16 +178,15 @@ const correct_answer_map: Record<string, any[]> = {
     null,
   ],
   date: [new Date("1990-01-01"), new Date("9999-12-31"), null],
-  timestamp_s: ["290309-12-22 (BC) 00:00:00", "294247-01-10 04:00:54", null],
-
-  timestamp_ns: [
-    "1677-09-21 00:12:43.145225",
-    "2262-04-11 23:47:16.854775",
+  timestamp_s: [
+    new Date(Date.UTC(1990, 0, 1)),
+    new Date("9999-12-31T23:59:59.000Z"),
     null,
   ],
+  timestamp_ns: [new Date(Date.UTC(1990, 0, 1)), null],
   timestamp_ms: [
-    "290309-12-22 (BC) 00:00:00",
-    "294247-01-10 04:00:54.775",
+    new Date(Date.UTC(1990, 0, 1)),
+    new Date("9999-12-31T23:59:59.000Z"),
     null,
   ],
   timestamp_tz: [

package/test/tokenize.test.ts

@@ -0,0 +1,74 @@
+import * as assert from 'assert';
+import * as duckdb from '..';
+
+describe('tokenize', function () {
+  it('should return correct tokens for a single statement', function () {
+    const db = new duckdb.Database(':memory:');
+    const output = db.tokenize('select 1;');
+    assert.deepStrictEqual(output, {
+      offsets: [0, 7, 8],
+      types: [duckdb.TokenType.KEYWORD, duckdb.TokenType.NUMERIC_CONSTANT, duckdb.TokenType.OPERATOR]
+    });
+  });
+  it('should return correct tokens for a multiple statements', function () {
+    const db = new duckdb.Database(':memory:');
+    const output = db.tokenize('select 1; select 2;');
+    assert.deepStrictEqual(output, {
+      offsets: [0, 7, 8, 10, 17, 18],
+      types: [
+        duckdb.TokenType.KEYWORD, duckdb.TokenType.NUMERIC_CONSTANT, duckdb.TokenType.OPERATOR,
+        duckdb.TokenType.KEYWORD, duckdb.TokenType.NUMERIC_CONSTANT, duckdb.TokenType.OPERATOR
+      ]
+    });
+  });
+  it('should return no tokens for an empty string', function () {
+    const db = new duckdb.Database(':memory:');
+    const output = db.tokenize('');
+    assert.deepStrictEqual(output, {
+      offsets: [],
+      types: []
+    });
+  });
+  it('should handle quoted semicolons in string constants', function () {
+    const db = new duckdb.Database(':memory:');
+    const output = db.tokenize(`select ';';`);
+    assert.deepStrictEqual(output, {
+      offsets: [0, 7, 10],
+      types: [duckdb.TokenType.KEYWORD, duckdb.TokenType.STRING_CONSTANT, duckdb.TokenType.OPERATOR]
+    });
+  });
+  it('should handle quoted semicolons in identifiers', function () {
+    const db = new duckdb.Database(':memory:');
+    const output = db.tokenize(`from ";";`);
+    assert.deepStrictEqual(output, {
+      offsets: [0, 5, 8],
+      types: [duckdb.TokenType.KEYWORD, duckdb.TokenType.IDENTIFIER, duckdb.TokenType.OPERATOR]
+    });
+  });
+  it('should handle comments', function () {
+    const db = new duckdb.Database(':memory:');
+    const output = db.tokenize(`select /* comment */ 1`);
+    // Note that the tokenizer doesn't return tokens for comments.
+    assert.deepStrictEqual(output, {
+      offsets: [0, 21],
+      types: [duckdb.TokenType.KEYWORD, duckdb.TokenType.NUMERIC_CONSTANT]
+    });
+  });
+  it('should handle invalid syntax', function () {
+    const db = new duckdb.Database(':memory:');
+    const output = db.tokenize(`selec 1`);
+    // The misspelled keyword is scanned as an identifier.
+    assert.deepStrictEqual(output, {
+      offsets: [0, 6],
+      types: [duckdb.TokenType.IDENTIFIER, duckdb.TokenType.NUMERIC_CONSTANT]
+    });
+  });
+  it('should support inverse TokenType mapping', function () {
+    assert.equal(duckdb.TokenType[duckdb.TokenType.IDENTIFIER], "IDENTIFIER");
+    assert.equal(duckdb.TokenType[duckdb.TokenType.NUMERIC_CONSTANT], "NUMERIC_CONSTANT");
+    assert.equal(duckdb.TokenType[duckdb.TokenType.STRING_CONSTANT], "STRING_CONSTANT");
+    assert.equal(duckdb.TokenType[duckdb.TokenType.OPERATOR], "OPERATOR");
+    assert.equal(duckdb.TokenType[duckdb.TokenType.KEYWORD], "KEYWORD");
+    assert.equal(duckdb.TokenType[duckdb.TokenType.COMMENT], "COMMENT");
+  });
+});

package/{configure → vendor}

@@ -5,4 +5,4 @@ set -x
 
 cd `dirname $0`
 
-python3 configure.py
+python3 vendor.py $@