duckdb 0.6.2-dev712.0 → 0.6.2-dev733.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.6.2-dev712.0",
+  "version": "0.6.2-dev733.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/src/execution/operator/projection/physical_tableinout_function.cpp

@@ -4,10 +4,13 @@ namespace duckdb {
 
 class TableInOutLocalState : public OperatorState {
 public:
-	TableInOutLocalState() {
+	TableInOutLocalState() : row_index(0), new_row(true) {
 	}
 
 	unique_ptr<LocalTableFunctionState> local_state;
+	idx_t row_index;
+	bool new_row;
+	DataChunk input_chunk;
 };
 
 class TableInOutGlobalState : public GlobalOperatorState {
@@ -20,9 +23,11 @@ public:
 
 PhysicalTableInOutFunction::PhysicalTableInOutFunction(vector<LogicalType> types, TableFunction function_p,
                                                        unique_ptr<FunctionData> bind_data_p,
-                                                       vector<column_t> column_ids_p, idx_t estimated_cardinality)
+                                                       vector<column_t> column_ids_p, idx_t estimated_cardinality,
+                                                       vector<column_t> project_input_p)
     : PhysicalOperator(PhysicalOperatorType::INOUT_FUNCTION, move(types), estimated_cardinality),
-      function(move(function_p)), bind_data(move(bind_data_p)), column_ids(move(column_ids_p)) {
+      function(move(function_p)), bind_data(move(bind_data_p)), column_ids(move(column_ids_p)),
+      projected_input(move(project_input_p)) {
 }
 
 unique_ptr<OperatorState> PhysicalTableInOutFunction::GetOperatorState(ExecutionContext &context) const {
@@ -32,6 +37,9 @@ unique_ptr<OperatorState> PhysicalTableInOutFunction::GetOperatorState(Execution
 		TableFunctionInitInput input(bind_data.get(), column_ids, vector<idx_t>(), nullptr);
 		result->local_state = function.init_local(context, input, gstate.global_state.get());
 	}
+	if (!projected_input.empty()) {
+		result->input_chunk.Initialize(context.client, children[0]->types);
+	}
 	return move(result);
 }
 
@@ -49,7 +57,46 @@ OperatorResultType PhysicalTableInOutFunction::Execute(ExecutionContext &context
 	auto &gstate = (TableInOutGlobalState &)gstate_p;
 	auto &state = (TableInOutLocalState &)state_p;
 	TableFunctionInput data(bind_data.get(), state.local_state.get(), gstate.global_state.get());
-	return function.in_out_function(context, data, input, chunk);
+	if (projected_input.empty()) {
+		// straightforward case - no need to project input
+		return function.in_out_function(context, data, input, chunk);
+	}
+	// when project_input is set we execute the input function row-by-row
+	if (state.new_row) {
+		if (state.row_index >= input.size()) {
+			// finished processing this chunk
+			state.new_row = true;
+			state.row_index = 0;
+			return OperatorResultType::NEED_MORE_INPUT;
+		}
+		// we are processing a new row: fetch the data for the current row
+		D_ASSERT(input.ColumnCount() == state.input_chunk.ColumnCount());
+		// set up the input data to the table in-out function
+		for (idx_t col_idx = 0; col_idx < input.ColumnCount(); col_idx++) {
+			ConstantVector::Reference(state.input_chunk.data[col_idx], input.data[col_idx], state.row_index, 1);
+		}
+		state.input_chunk.SetCardinality(1);
+		state.row_index++;
+		state.new_row = false;
+	}
+	// set up the output data in "chunk"
+	D_ASSERT(chunk.ColumnCount() > projected_input.size());
+	D_ASSERT(state.row_index > 0);
+	idx_t base_idx = chunk.ColumnCount() - projected_input.size();
+	for (idx_t project_idx = 0; project_idx < projected_input.size(); project_idx++) {
+		auto source_idx = projected_input[project_idx];
+		auto target_idx = base_idx + project_idx;
+		ConstantVector::Reference(chunk.data[target_idx], input.data[source_idx], state.row_index - 1, 1);
+	}
+	auto result = function.in_out_function(context, data, state.input_chunk, chunk);
+	if (result == OperatorResultType::FINISHED) {
+		return result;
+	}
+	if (result == OperatorResultType::NEED_MORE_INPUT) {
+		// we finished processing this row: move to the next row
+		state.new_row = true;
+	}
+	return OperatorResultType::HAVE_MORE_OUTPUT;
 }
 
 OperatorFinalizeResultType PhysicalTableInOutFunction::FinalExecute(ExecutionContext &context, DataChunk &chunk,
@@ -57,6 +104,9 @@ OperatorFinalizeResultType PhysicalTableInOutFunction::FinalExecute(ExecutionCon
                                                                     OperatorState &state_p) const {
 	auto &gstate = (TableInOutGlobalState &)gstate_p;
 	auto &state = (TableInOutLocalState &)state_p;
+	if (!projected_input.empty()) {
+		throw InternalException("FinalExecute not supported for project_input");
+	}
 	TableFunctionInput data(bind_data.get(), state.local_state.get(), gstate.global_state.get());
 	return function.in_out_function_final(context, data, chunk);
 }

package/src/duckdb/src/execution/operator/projection/physical_unnest.cpp

@@ -89,6 +89,7 @@ static void UnnestValidity(UnifiedVectorFormat &vdata, idx_t start, idx_t end, V
 
 static void UnnestVector(UnifiedVectorFormat &vdata, Vector &source, idx_t list_size, idx_t start, idx_t end,
                          Vector &result) {
+	D_ASSERT(source.GetType() == result.GetType());
 	switch (result.GetType().InternalType()) {
 	case PhysicalType::BOOL:
 	case PhysicalType::INT8:

package/src/duckdb/src/execution/physical_plan/plan_get.cpp

@@ -33,11 +33,14 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalGet &op) {
 	if (!op.children.empty()) {
 		// this is for table producing functions that consume subquery results
 		D_ASSERT(op.children.size() == 1);
-		auto node = make_unique<PhysicalTableInOutFunction>(op.returned_types, op.function, move(op.bind_data),
-		                                                    op.column_ids, op.estimated_cardinality);
+		auto node = make_unique<PhysicalTableInOutFunction>(op.types, op.function, move(op.bind_data), op.column_ids,
+		                                                    op.estimated_cardinality, move(op.projected_input));
 		node->children.push_back(CreatePlan(move(op.children[0])));
 		return move(node);
 	}
+	if (!op.projected_input.empty()) {
+		throw InternalException("LogicalGet::project_input can only be set for table-in-out functions");
+	}
 
 	unique_ptr<TableFilterSet> table_filters;
 	if (!op.table_filters.filters.empty()) {

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.6.2-dev712"
+#define DUCKDB_VERSION "0.6.2-dev733"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "4db88c5e6b"
+#define DUCKDB_SOURCE_ID "cbe7a7fdc7"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/common/field_writer.hpp

@@ -203,6 +203,21 @@ public:
 		return source.Read<T>();
 	}
 
+	template <class T, class CONTAINER_TYPE = vector<T>>
+	bool ReadList(CONTAINER_TYPE &result) {
+		if (field_count >= max_field_count) {
+			// field is not there, return false and leave the result empty
+			return false;
+		}
+		AddField();
+		auto result_count = source.Read<uint32_t>();
+		result.reserve(result_count);
+		for (idx_t i = 0; i < result_count; i++) {
+			result.push_back(source.Read<T>());
+		}
+		return true;
+	}
+
 	template <class T, class CONTAINER_TYPE = vector<T>>
 	CONTAINER_TYPE ReadRequiredList() {
 		if (field_count >= max_field_count) {

package/src/duckdb/src/include/duckdb/execution/operator/projection/physical_tableinout_function.hpp

@@ -19,7 +19,7 @@ class PhysicalTableInOutFunction : public PhysicalOperator {
 public:
 	PhysicalTableInOutFunction(vector<LogicalType> types, TableFunction function_p,
 	                           unique_ptr<FunctionData> bind_data_p, vector<column_t> column_ids_p,
-	                           idx_t estimated_cardinality);
+	                           idx_t estimated_cardinality, vector<column_t> projected_input);
 
 public:
 	unique_ptr<OperatorState> GetOperatorState(ExecutionContext &context) const override;
@@ -42,8 +42,10 @@ private:
 	TableFunction function;
 	//! Bind data of the function
 	unique_ptr<FunctionData> bind_data;
-
+	//! The set of column ids to fetch
 	vector<column_t> column_ids;
+	//! The set of input columns to project out
+	vector<column_t> projected_input;
 };
 
 } // namespace duckdb

package/src/duckdb/src/include/duckdb/main/extension_helper.hpp

@@ -57,6 +57,9 @@ public:
 	static void ReplacementOpenPost(ClientContext &context, const string &extension, DatabaseInstance &instance,
 	                                ReplacementOpenData *open_data);
 
+	// Returns extension name, or empty string if not a replacement open path
+	static string ExtractExtensionPrefixFromPath(const string &path);
+
 private:
 	static const vector<string> PathComponents();
 	static ExtensionInitResult InitialLoad(DBConfig &context, FileOpener *opener, const string &extension);

package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp

@@ -44,6 +44,8 @@ public:
 	vector<LogicalType> input_table_types;
 	//! The set of named input table names for the table-in table-out function
 	vector<string> input_table_names;
+	//! For a table-in-out function, the set of projected input columns
+	vector<column_t> projected_input;
 
 	string GetName() const override;
 	string ParamsToString() const override;

package/src/duckdb/src/main/db_instance_cache.cpp

@@ -1,5 +1,5 @@
 #include "duckdb/main/db_instance_cache.hpp"
-
+#include "duckdb/main/extension_helper.hpp"
 namespace duckdb {
 
 string GetDBAbsolutePath(const string &database) {
@@ -10,6 +10,10 @@ string GetDBAbsolutePath(const string &database) {
 		// this is a memory db, just return it.
 		return database;
 	}
+	if (!ExtensionHelper::ExtractExtensionPrefixFromPath(database).empty()) {
+		// this database path is handled by a replacement open and is not a file path
+		return database;
+	}
 	if (FileSystem::IsPathAbsolute(database)) {
 		return database;
 	}

package/src/duckdb/src/main/extension/extension_load.cpp

@@ -176,4 +176,20 @@ void ExtensionHelper::ReplacementOpenPost(ClientContext &context, const string &
 	}
 }
 
+string ExtensionHelper::ExtractExtensionPrefixFromPath(const string &path) {
+	auto first_colon = path.find(':');
+	if (first_colon == string::npos || first_colon < 2) { // needs to be at least two characters because windows c: ...
+		return "";
+	}
+	auto extension = path.substr(0, first_colon);
+	D_ASSERT(extension.size() > 1);
+	// needs to be alphanumeric
+	for (auto &ch : extension) {
+		if (!isalnum(ch) && ch != '_') {
+			return "";
+		}
+	}
+	return extension;
+}
+
 } // namespace duckdb

package/src/duckdb/src/main/extension_prefix_opener.cpp

@@ -16,18 +16,10 @@ struct ExtensionPrefixOpenData : public ReplacementOpenData {
 
 static unique_ptr<ReplacementOpenData> ExtensionPrefixPreOpen(DBConfig &config, ReplacementOpenStaticData *) {
 	auto path = config.options.database_path;
-	auto first_colon = path.find(':');
-	if (first_colon == string::npos || first_colon < 2) { // needs to be at least two characters because windows c: ...
+	string extension = ExtensionHelper::ExtractExtensionPrefixFromPath(path);
+	if (extension.empty()) {
 		return nullptr;
 	}
-	auto extension = path.substr(0, first_colon);
-	D_ASSERT(extension.size() > 1);
-	// needs to be alphanumeric
-	for (auto &ch : extension) {
-		if (!isalnum(ch) && ch != '_') {
-			return nullptr;
-		}
-	}
 	auto extension_data = ExtensionHelper::ReplacementOpenPre(extension, config);
 	if (extension_data) {
 		return make_unique<ExtensionPrefixOpenData>(extension, path, move(extension_data));

package/src/duckdb/src/optimizer/remove_unused_columns.cpp

@@ -215,6 +215,9 @@ void RemoveUnusedColumns::VisitOperator(LogicalOperator &op) {
 		LogicalOperatorVisitor::VisitOperatorExpressions(op);
 		if (!everything_referenced) {
 			auto &get = (LogicalGet &)op;
+			if (!get.function.projection_pushdown) {
+				return;
+			}
 
 			// Create "selection vector" of all column ids
 			vector<idx_t> proj_sel;

package/src/duckdb/src/parallel/pipeline_executor.cpp

@@ -133,8 +133,11 @@ void PipelineExecutor::FlushCachingOperatorsPull(DataChunk &result) {
 			finalize_result = cached_final_execute_result;
 		} else {
 			// Flush the current operator
-			finalize_result = pipeline.operators[op_idx]->FinalExecute(
-			    context, curr_chunk, *pipeline.operators[op_idx]->op_state, *intermediate_states[op_idx]);
+			auto current_operator = pipeline.operators[op_idx];
+			StartOperator(current_operator);
+			finalize_result = current_operator->FinalExecute(context, curr_chunk, *current_operator->op_state,
+			                                                 *intermediate_states[op_idx]);
+			EndOperator(current_operator, &curr_chunk);
 		}
 
 		auto execute_result = Execute(curr_chunk, result, op_idx + 1);
@@ -171,8 +174,11 @@ void PipelineExecutor::FlushCachingOperatorsPush() {
 		do {
 			auto &curr_chunk =
 			    op_idx + 1 >= intermediate_chunks.size() ? final_chunk : *intermediate_chunks[op_idx + 1];
-			finalize_result = pipeline.operators[op_idx]->FinalExecute(
-			    context, curr_chunk, *pipeline.operators[op_idx]->op_state, *intermediate_states[op_idx]);
+			auto current_operator = pipeline.operators[op_idx];
+			StartOperator(current_operator);
+			finalize_result = current_operator->FinalExecute(context, curr_chunk, *current_operator->op_state,
+			                                                 *intermediate_states[op_idx]);
+			EndOperator(current_operator, &curr_chunk);
 			push_result = ExecutePushInternal(curr_chunk, op_idx + 1);
 		} while (finalize_result != OperatorFinalizeResultType::FINISHED &&
 		         push_result != OperatorResultType::FINISHED);

package/src/duckdb/src/parser/transform/tableref/transform_table_function.cpp

@@ -5,9 +5,6 @@
 namespace duckdb {
 
 unique_ptr<TableRef> Transformer::TransformRangeFunction(duckdb_libpgquery::PGRangeFunction *root) {
-	if (root->lateral) {
-		throw NotImplementedException("LATERAL not implemented");
-	}
 	if (root->ordinality) {
 		throw NotImplementedException("WITH ORDINALITY not implemented");
 	}

package/src/duckdb/src/planner/binder/tableref/bind_table_function.cpp

@@ -156,6 +156,12 @@ Binder::BindTableFunctionInternal(TableFunction &table_function, const string &f
 	get->named_parameters = named_parameters;
 	get->input_table_types = input_table_types;
 	get->input_table_names = input_table_names;
+	if (table_function.in_out_function && !table_function.projection_pushdown) {
+		get->column_ids.reserve(return_types.size());
+		for (idx_t i = 0; i < return_types.size(); i++) {
+			get->column_ids.push_back(i);
+		}
+	}
 	// now add the table function to the bind context so its columns can be bound
 	bind_context.AddTableFunction(bind_index, function_name, return_names, return_types, get->column_ids,
 	                              get->GetTable());

package/src/duckdb/src/planner/operator/logical_get.cpp

@@ -54,6 +54,16 @@ vector<ColumnBinding> LogicalGet::GetColumnBindings() {
 			result.emplace_back(table_index, proj_id);
 		}
 	}
+	if (!projected_input.empty()) {
+		if (children.size() != 1) {
+			throw InternalException("LogicalGet::project_input can only be set for table-in-out functions");
+		}
+		auto child_bindings = children[0]->GetColumnBindings();
+		for (auto entry : projected_input) {
+			D_ASSERT(entry < child_bindings.size());
+			result.emplace_back(child_bindings[entry]);
+		}
+	}
 	return result;
 }
 
@@ -80,6 +90,15 @@ void LogicalGet::ResolveTypes() {
 			}
 		}
 	}
+	if (!projected_input.empty()) {
+		if (children.size() != 1) {
+			throw InternalException("LogicalGet::project_input can only be set for table-in-out functions");
+		}
+		for (auto entry : projected_input) {
+			D_ASSERT(entry < children[0]->types.size());
+			types.push_back(children[0]->types[entry]);
+		}
+	}
 }
 
 idx_t LogicalGet::EstimateCardinality(ClientContext &context) {
@@ -113,6 +132,7 @@ void LogicalGet::Serialize(FieldWriter &writer) const {
 		writer.WriteRegularSerializableList(input_table_types);
 		writer.WriteList<string>(input_table_names);
 	}
+	writer.WriteList<column_t>(projected_input);
 }
 
 unique_ptr<LogicalOperator> LogicalGet::Deserialize(LogicalDeserializationState &state, FieldReader &reader) {
@@ -162,6 +182,8 @@ unique_ptr<LogicalOperator> LogicalGet::Deserialize(LogicalDeserializationState
 			    "Table function deserialization failure - bind returned different returned names than were serialized");
 		}
 	}
+	vector<column_t> projected_input;
+	reader.ReadList<column_t>(projected_input);
 
 	auto result = make_unique<LogicalGet>(table_index, function, move(bind_data), returned_types, returned_names);
 	result->column_ids = move(column_ids);
@@ -171,6 +193,7 @@ unique_ptr<LogicalOperator> LogicalGet::Deserialize(LogicalDeserializationState
 	result->named_parameters = move(named_parameters);
 	result->input_table_types = input_table_types;
 	result->input_table_names = input_table_names;
+	result->projected_input = move(projected_input);
 	return move(result);
 }
 

package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp

@@ -82,8 +82,8 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 		auto left_columns = plan->GetColumnBindings().size();
 		auto delim_index = binder.GenerateTableIndex();
 		this->base_binding = ColumnBinding(delim_index, 0);
-		this->delim_offset = 0;
-		this->data_offset = left_columns;
+		this->delim_offset = left_columns;
+		this->data_offset = 0;
 		auto delim_scan = make_unique<LogicalDelimGet>(delim_index, delim_types);
 		return LogicalCrossProduct::Create(move(plan), move(delim_scan));
 	}
@@ -505,8 +505,19 @@ unique_ptr<LogicalOperator> FlattenDependentJoins::PushDownDependentJoinInternal
 	case LogicalOperatorType::LOGICAL_ORDER_BY:
 		plan->children[0] = PushDownDependentJoin(move(plan->children[0]));
 		return plan;
-	case LogicalOperatorType::LOGICAL_GET:
-		throw BinderException("Table-in table-out functions not (yet) supported in correlated subqueries");
+	case LogicalOperatorType::LOGICAL_GET: {
+		auto &get = (LogicalGet &)*plan;
+		if (get.children.size() != 1) {
+			throw InternalException("Flatten dependent joins - logical get encountered without children");
+		}
+		plan->children[0] = PushDownDependentJoin(move(plan->children[0]));
+		for (idx_t i = 0; i < (perform_delim ? correlated_columns.size() : 1); i++) {
+			get.projected_input.push_back(this->delim_offset + i);
+		}
+		this->delim_offset = get.returned_types.size();
+		this->data_offset = 0;
+		return plan;
+	}
 	case LogicalOperatorType::LOGICAL_RECURSIVE_CTE: {
 		throw BinderException("Recursive CTEs not supported in correlated subquery");
 	}