duckdb 0.9.1-dev95.0 → 0.9.1

package/src/duckdb/src/planner/binder/expression/bind_comparison_expression.cpp

@@ -18,20 +18,25 @@
 
 namespace duckdb {
 
-unique_ptr<Expression> ExpressionBinder::PushCollation(ClientContext &context, unique_ptr<Expression> source,
-                                                       const string &collation_p, bool equality_only) {
+bool ExpressionBinder::PushCollation(ClientContext &context, unique_ptr<Expression> &source,
+                                     const LogicalType &sql_type, bool equality_only) {
+	if (sql_type.id() != LogicalTypeId::VARCHAR) {
+		// only VARCHAR columns require collation
+		return false;
+	}
 	// replace default collation with system collation
+	auto str_collation = StringType::GetCollation(sql_type);
 	string collation;
-	if (collation_p.empty()) {
+	if (str_collation.empty()) {
 		collation = DBConfig::GetConfig(context).options.collation;
 	} else {
-		collation = collation_p;
+		collation = str_collation;
 	}
 	collation = StringUtil::Lower(collation);
 	// bind the collation
 	if (collation.empty() || collation == "binary" || collation == "c" || collation == "posix") {
-		// binary collation: just skip
-		return source;
+		// no collation or binary collation: skip
+		return false;
 	}
 	auto &catalog = Catalog::GetSystemCatalog(context);
 	auto splits = StringUtil::Split(StringUtil::Lower(collation), ".");
@@ -60,11 +65,12 @@ unique_ptr<Expression> ExpressionBinder::PushCollation(ClientContext &context, u
 		auto function = function_binder.BindScalarFunction(collation_entry.function, std::move(children));
 		source = std::move(function);
 	}
-	return source;
+	return true;
 }
 
 void ExpressionBinder::TestCollation(ClientContext &context, const string &collation) {
-	PushCollation(context, make_uniq<BoundConstantExpression>(Value("")), collation);
+	auto expr = make_uniq_base<Expression, BoundConstantExpression>(Value(""));
+	PushCollation(context, expr, LogicalType::VARCHAR_COLLATION(collation));
 }
 
 LogicalType BoundComparisonExpression::BindComparison(LogicalType left_type, LogicalType right_type) {
@@ -134,12 +140,9 @@ BindResult ExpressionBinder::BindExpression(ComparisonExpression &expr, idx_t de
 	right = BoundCastExpression::AddCastToType(context, std::move(right), input_type,
 	                                           input_type.id() == LogicalTypeId::ENUM);
 
-	if (input_type.id() == LogicalTypeId::VARCHAR) {
-		// handle collation
-		auto collation = StringType::GetCollation(input_type);
-		left = PushCollation(context, std::move(left), collation, expr.type == ExpressionType::COMPARE_EQUAL);
-		right = PushCollation(context, std::move(right), collation, expr.type == ExpressionType::COMPARE_EQUAL);
-	}
+	PushCollation(context, left, input_type, expr.type == ExpressionType::COMPARE_EQUAL);
+	PushCollation(context, right, input_type, expr.type == ExpressionType::COMPARE_EQUAL);
+
 	// now create the bound comparison expression
 	return BindResult(make_uniq<BoundComparisonExpression>(expr.type, std::move(left), std::move(right)));
 }

package/src/duckdb/src/planner/binder/query_node/bind_select_node.cpp

@@ -222,10 +222,7 @@ void Binder::BindModifierTypes(BoundQueryNode &result, const vector<LogicalType>
 			for (auto &target_distinct : distinct.target_distincts) {
 				auto &bound_colref = target_distinct->Cast<BoundColumnRefExpression>();
 				const auto &sql_type = sql_types[bound_colref.binding.column_index];
-				if (sql_type.id() == LogicalTypeId::VARCHAR) {
-					target_distinct = ExpressionBinder::PushCollation(context, std::move(target_distinct),
-					                                                  StringType::GetCollation(sql_type), true);
-				}
+				ExpressionBinder::PushCollation(context, target_distinct, sql_type, true);
 			}
 			break;
 		}
@@ -253,10 +250,7 @@ void Binder::BindModifierTypes(BoundQueryNode &result, const vector<LogicalType>
 				D_ASSERT(bound_colref.binding.column_index < sql_types.size());
 				const auto &sql_type = sql_types[bound_colref.binding.column_index];
 				bound_colref.return_type = sql_types[bound_colref.binding.column_index];
-				if (sql_type.id() == LogicalTypeId::VARCHAR) {
-					order_node.expression = ExpressionBinder::PushCollation(context, std::move(order_node.expression),
-					                                                        StringType::GetCollation(sql_type));
-				}
+				ExpressionBinder::PushCollation(context, order_node.expression, sql_type);
 			}
 			break;
 		}
@@ -389,9 +383,8 @@ unique_ptr<BoundQueryNode> Binder::BindSelectNode(SelectNode &statement, unique_
 			bool contains_subquery = bound_expr_ref.HasSubquery();
 
 			// push a potential collation, if necessary
-			auto collated_expr = ExpressionBinder::PushCollation(context, std::move(bound_expr),
-			                                                     StringType::GetCollation(group_type), true);
-			if (!contains_subquery && !collated_expr->Equals(bound_expr_ref)) {
+			bool requires_collation = ExpressionBinder::PushCollation(context, bound_expr, group_type, true);
+			if (!contains_subquery && requires_collation) {
 				// if there is a collation on a group x, we should group by the collated expr,
 				// but also push a first(x) aggregate in case x is selected (uncollated)
 				info.collated_groups[i] = result->aggregates.size();
@@ -405,7 +398,7 @@ unique_ptr<BoundQueryNode> Binder::BindSelectNode(SelectNode &statement, unique_
 				auto function = function_binder.BindAggregateFunction(first_fun, std::move(first_children));
 				result->aggregates.push_back(std::move(function));
 			}
-			result->groups.group_expressions.push_back(std::move(collated_expr));
+			result->groups.group_expressions.push_back(std::move(bound_expr));
 
 			// in the unbound expression we DO bind the table names of any ColumnRefs
 			// we do this to make sure that "table.a" and "a" are treated the same

package/src/duckdb/src/planner/binder/statement/bind_create.cpp

@@ -68,6 +68,16 @@ void Binder::BindSchemaOrCatalog(string &catalog, string &schema) {
 	BindSchemaOrCatalog(context, catalog, schema);
 }
 
+const string Binder::BindCatalog(string &catalog) {
+	auto &db_manager = DatabaseManager::Get(context);
+	optional_ptr<AttachedDatabase> database = db_manager.GetDatabase(context, catalog);
+	if (database) {
+		return db_manager.GetDatabase(context, catalog).get()->GetName();
+	} else {
+		return db_manager.GetDefaultDatabase(context);
+	}
+}
+
 SchemaCatalogEntry &Binder::BindSchema(CreateInfo &info) {
 	BindSchemaOrCatalog(info.catalog, info.schema);
 	if (IsInvalidCatalog(info.catalog) && info.temporary) {
@@ -456,9 +466,13 @@ BoundStatement Binder::Bind(CreateStatement &stmt) {
 
 	auto catalog_type = stmt.info->type;
 	switch (catalog_type) {
-	case CatalogType::SCHEMA_ENTRY:
+	case CatalogType::SCHEMA_ENTRY: {
+		auto &base = stmt.info->Cast<CreateInfo>();
+		auto catalog = BindCatalog(base.catalog);
+		properties.modified_databases.insert(catalog);
 		result.plan = make_uniq<LogicalCreate>(LogicalOperatorType::LOGICAL_CREATE_SCHEMA, std::move(stmt.info));
 		break;
+	}
 	case CatalogType::VIEW_ENTRY: {
 		auto &base = stmt.info->Cast<CreateViewInfo>();
 		// bind the schema

package/src/duckdb/src/planner/bound_parameter_map.cpp

@@ -43,19 +43,30 @@ shared_ptr<BoundParameterData> BoundParameterMap::CreateOrGetData(const string &
 }
 
 unique_ptr<BoundParameterExpression> BoundParameterMap::BindParameterExpression(ParameterExpression &expr) {
-	auto &identifier = expr.identifier;
-	auto return_type = GetReturnType(identifier);
 
+	auto &identifier = expr.identifier;
 	D_ASSERT(!parameter_data.count(identifier));
 
 	// No value has been supplied yet,
-	// We return a shared pointer to an object that will get populated wtih a Value later
-	// When the BoundParameterExpression get executed, this will be used to get the corresponding value
+	// We return a shared pointer to an object that will get populated with a Value later
+	// When the BoundParameterExpression gets executed, this will be used to get the corresponding value
 	auto param_data = CreateOrGetData(identifier);
 	auto bound_expr = make_uniq<BoundParameterExpression>(identifier);
+
 	bound_expr->parameter_data = param_data;
-	bound_expr->return_type = return_type;
 	bound_expr->alias = expr.alias;
+
+	auto param_type = param_data->return_type;
+	auto identifier_type = GetReturnType(identifier);
+
+	// we found a type for this bound parameter, but now we found another occurrence with the same identifier,
+	// a CAST around this consecutive occurrence might swallow the unknown type of this consecutive occurrence,
+	// then, if we do not rebind, we potentially have unknown data types during execution
+	if (identifier_type == LogicalType::UNKNOWN && param_type != LogicalType::UNKNOWN) {
+		rebind = true;
+	}
+
+	bound_expr->return_type = identifier_type;
 	return bound_expr;
 }
 

package/src/duckdb/src/planner/expression_binder/base_select_binder.cpp

@@ -98,11 +98,8 @@ BindResult BaseSelectBinder::BindColumnRef(unique_ptr<ParsedExpression> &expr_pt
 				                      " This is not yet supported.",
 				                      colref.column_names[0]);
 			}
-			auto result = BindResult(node.select_list[index]->Copy());
-			if (result.expression->type == ExpressionType::BOUND_COLUMN_REF) {
-				auto &result_expr = result.expression->Cast<BoundColumnRefExpression>();
-				result_expr.depth = depth;
-			}
+			auto copied_expression = node.original_expressions[index]->Copy();
+			result = BindExpression(copied_expression, depth, false);
 			return result;
 		}
 	}

package/src/duckdb/src/planner/planner.cpp

@@ -76,7 +76,7 @@ void Planner::CreatePlan(SQLStatement &statement) {
 	}
 	this->properties = binder->properties;
 	this->properties.parameter_count = parameter_count;
-	properties.bound_all_parameters = parameters_resolved;
+	properties.bound_all_parameters = !bound_parameters.rebind && parameters_resolved;
 
 	Planner::VerifyPlan(context, plan, bound_parameters.GetParametersPtr());
 

package/src/duckdb/src/transaction/duck_transaction_manager.cpp

@@ -252,6 +252,7 @@ void DuckTransactionManager::RollbackTransaction(Transaction *transaction_p) {
 }
 
 void DuckTransactionManager::RemoveTransaction(DuckTransaction &transaction) noexcept {
+	bool changes_made = transaction.ChangesMade();
 	// remove the transaction from the list of active transactions
 	idx_t t_index = active_transactions.size();
 	// check for the lowest and highest start time in the list of transactions
@@ -275,15 +276,18 @@ void DuckTransactionManager::RemoveTransaction(DuckTransaction &transaction) noe
 	D_ASSERT(t_index != active_transactions.size());
 	auto current_transaction = std::move(active_transactions[t_index]);
 	auto current_query = DatabaseManager::Get(db).ActiveQueryNumber();
-	if (transaction.commit_id != 0) {
-		// the transaction was committed, add it to the list of recently
-		// committed transactions
-		recently_committed_transactions.push_back(std::move(current_transaction));
-	} else {
-		// the transaction was aborted, but we might still need its information
-		// add it to the set of transactions awaiting GC
-		current_transaction->highest_active_query = current_query;
-		old_transactions.push_back(std::move(current_transaction));
+	if (changes_made) {
+		// if the transaction made any changes we need to keep it around
+		if (transaction.commit_id != 0) {
+			// the transaction was committed, add it to the list of recently
+			// committed transactions
+			recently_committed_transactions.push_back(std::move(current_transaction));
+		} else {
+			// the transaction was aborted, but we might still need its information
+			// add it to the set of transactions awaiting GC
+			current_transaction->highest_active_query = current_query;
+			old_transactions.push_back(std::move(current_transaction));
+		}
 	}
 	// remove the transaction from the set of currently active transactions
 	active_transactions.erase(active_transactions.begin() + t_index);

package/src/duckdb/third_party/parquet/parquet_types.h

@@ -92,7 +92,8 @@ struct Encoding {
     DELTA_BINARY_PACKED = 5,
     DELTA_LENGTH_BYTE_ARRAY = 6,
     DELTA_BYTE_ARRAY = 7,
-    RLE_DICTIONARY = 8
+    RLE_DICTIONARY = 8,
+    BYTE_STREAM_SPLIT = 9,
   };
 };
 

package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp

@@ -348,17 +348,17 @@
 
 #include "extension/icu/third_party/icu/i18n/wintzimpl.cpp"
 
-#include "extension/icu/third_party/icu/i18n/double-conversion-strtod.cpp"
+#include "extension/icu/third_party/icu/i18n/double-conversion-fast-dtoa.cpp"
 
-#include "extension/icu/third_party/icu/i18n/double-conversion-double-to-string.cpp"
+#include "extension/icu/third_party/icu/i18n/double-conversion-cached-powers.cpp"
 
-#include "extension/icu/third_party/icu/i18n/double-conversion-fast-dtoa.cpp"
+#include "extension/icu/third_party/icu/i18n/double-conversion-strtod.cpp"
 
-#include "extension/icu/third_party/icu/i18n/double-conversion-string-to-double.cpp"
+#include "extension/icu/third_party/icu/i18n/double-conversion-double-to-string.cpp"
 
 #include "extension/icu/third_party/icu/i18n/double-conversion-bignum-dtoa.cpp"
 
 #include "extension/icu/third_party/icu/i18n/double-conversion-bignum.cpp"
 
-#include "extension/icu/third_party/icu/i18n/double-conversion-cached-powers.cpp"
+#include "extension/icu/third_party/icu/i18n/double-conversion-string-to-double.cpp"
 

package/src/duckdb/ub_src_optimizer_pushdown.cpp

@@ -2,6 +2,8 @@
 
 #include "src/optimizer/pushdown/pushdown_cross_product.cpp"
 
+#include "src/optimizer/pushdown/pushdown_distinct.cpp"
+
 #include "src/optimizer/pushdown/pushdown_filter.cpp"
 
 #include "src/optimizer/pushdown/pushdown_get.cpp"

package/src/statement.cpp

@@ -377,10 +377,10 @@ struct RunPreparedTask : public Task {
 					// query results, the string data is owned by the QueryResult
 					auto result_ref_ptr = new std::shared_ptr<duckdb::QueryResult>(result_ptr);
 
-					auto array_buffer =
-					    Napi::ArrayBuffer::New(env, (void *)blob.GetData(), blob.GetSize(), deleter, result_ref_ptr);
+					auto array_buffer = Napi::Buffer<char>::NewOrCopy(env, (char *)blob.GetData(), blob.GetSize(),
+					                                                  deleter, result_ref_ptr);
 
-					auto typed_array = Napi::Uint8Array::New(env, blob.GetSize(), array_buffer, 0);
+					auto typed_array = Napi::TypedArrayOf<char>(env, array_buffer);
 
 					// TODO we should handle this in duckdb probably
 					if (is_header) {
@@ -708,7 +708,7 @@ struct GetNextArrowIpcTask : public Task {
 			delete static_cast<unique_ptr<duckdb::DataChunk> *>(hint);
 		};
 		auto array_buffer =
-		    Napi::ArrayBuffer::New(env, (void *)blob.GetData(), blob.GetSize(), deleter, data_chunk_ptr);
+		    Napi::Buffer<char>::NewOrCopy(env, (char *)blob.GetData(), blob.GetSize(), deleter, data_chunk_ptr);
 
 		deferred.Resolve(array_buffer);
 	}

package/test/arrow.test.ts

@@ -9,7 +9,9 @@ describe('arrow IPC API fails neatly when extension not loaded', function() {
     let conn;
     before((done) => {
         db = new duckdb.Database(':memory:', {"allow_unsigned_extensions": "true"}, () => {
-            done();
+            db.all('SET autoload_known_extensions=false;', () => {
+                done();
+            });
         });
     });
 

package/test/parquet.test.ts

@@ -8,7 +8,7 @@ describe('can query parquet', function() {
     });
 
     it('should be able to read parquet files', function(done) {
-        db.run("select * from parquet_scan('../../data/parquet-testing/userdata1.parquet')", done);
+        db.run("select * from parquet_scan('test/userdata1.parquet')", done);
     });
 
 });

package/{configure → vendor}

@@ -5,4 +5,4 @@ set -x
 
 cd `dirname $0`
 
-python3 configure.py
+python3 vendor.py $@

package/{configure.py → vendor.py}

@@ -2,6 +2,17 @@ import os
 import sys
 import json
 import pickle
+import argparse
+
+parser = argparse.ArgumentParser(description='Inlines DuckDB Sources')
+
+parser.add_argument('--duckdb', action='store',
+                    help='Path to the DuckDB Version to be vendored in', required=True, type=str)
+
+
+
+args = parser.parse_args()
+
 
 # list of extensions to bundle
 extensions = ['parquet', 'icu', 'json']
@@ -14,7 +25,7 @@ gyp_out = os.path.join(basedir, 'binding.gyp')
 cache_file = os.path.join(basedir, 'filelist.cache')
 
 # path to package_build.py
-os.chdir(os.path.join('..', '..'))
+os.chdir(os.path.join(args.duckdb))
 scripts_dir = 'scripts'
 
 sys.path.append(scripts_dir)

package/duckdb_extension_config.cmake

@@ -1,10 +0,0 @@
-################################################################################
-# NodeJS DuckDB extension config
-################################################################################
-#
-# This is the default extension configuration for NodeJS builds. Basically it means that all these extensions are
-# "baked in" to the NodeJS binaries Note that the configuration here is only when building Node using the main
-# CMakeLists.txt file with the `BUILD_NODE` variable.
-# TODO: unify this by making setup.py also use this configuration, making this the config for all Node builds
-duckdb_extension_load(json)
-duckdb_extension_load(icu)