duckdb 0.5.2-dev494.0 → 0.5.2-dev512.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "duckdb",
   "main": "./lib/duckdb.js",
-  "version": "0.5.2-dev494.0",
+  "version": "0.5.2-dev512.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb.cpp

@@ -42766,7 +42766,7 @@ void DataChunk::Fuse(DataChunk &other) {
 	other.Destroy();
 }
 
-void DataChunk::ReferenceColumns(DataChunk &other, vector<column_t> column_ids) {
+void DataChunk::ReferenceColumns(DataChunk &other, const vector<column_t> &column_ids) {
 	D_ASSERT(ColumnCount() == column_ids.size());
 	Reset();
 	for (idx_t col_idx = 0; col_idx < ColumnCount(); col_idx++) {
@@ -72428,16 +72428,25 @@ namespace duckdb {
 //! Represents a scan of a base table
 class PhysicalTableScan : public PhysicalOperator {
 public:
+	//! Regular Table Scan
 	PhysicalTableScan(vector<LogicalType> types, TableFunction function, unique_ptr<FunctionData> bind_data,
 	                  vector<column_t> column_ids, vector<string> names, unique_ptr<TableFilterSet> table_filters,
 	                  idx_t estimated_cardinality);
+	//! Table scan that immediately projects out filter columns that are unused in the remainder of the query plan
+	PhysicalTableScan(vector<LogicalType> types, TableFunction function, unique_ptr<FunctionData> bind_data,
+	                  vector<LogicalType> returned_types, vector<column_t> column_ids, vector<idx_t> projection_ids,
+	                  vector<string> names, unique_ptr<TableFilterSet> table_filters, idx_t estimated_cardinality);
 
 	//! The table function
 	TableFunction function;
 	//! Bind data of the function
 	unique_ptr<FunctionData> bind_data;
-	//! The projected-out column ids
+	//! The types of ALL columns that can be returned by the table function
+	vector<LogicalType> returned_types;
+	//! The column ids used within the table function
 	vector<column_t> column_ids;
+	//! The projected-out column ids
+	vector<idx_t> projection_ids;
 	//! The names of the columns
 	vector<string> names;
 	//! The table filters
@@ -74665,11 +74674,7 @@ SinkResultType PhysicalOrder::Sink(ExecutionContext &context, GlobalSinkState &g
 	lstate.key_executor.Execute(input, keys);
 
 	auto &payload = lstate.payload;
-	payload.Reset();
-	for (idx_t col_idx = 0; col_idx < projections.size(); col_idx++) {
-		payload.data[col_idx].Reference(input.data[projections[col_idx]]);
-	}
-	payload.SetCardinality(input.size());
+	payload.ReferenceColumns(input, projections);
 
 	// Sink the data into the local sort state
 	keys.Verify();
@@ -78642,7 +78647,7 @@ unique_ptr<OperatorState> PhysicalTableInOutFunction::GetOperatorState(Execution
 	auto &gstate = (TableInOutGlobalState &)*op_state;
 	auto result = make_unique<TableInOutLocalState>();
 	if (function.init_local) {
-		TableFunctionInitInput input(bind_data.get(), column_ids, nullptr);
+		TableFunctionInitInput input(bind_data.get(), column_ids, vector<idx_t>(), nullptr);
 		result->local_state = function.init_local(context, input, gstate.global_state.get());
 	}
 	return move(result);
@@ -78651,7 +78656,7 @@ unique_ptr<OperatorState> PhysicalTableInOutFunction::GetOperatorState(Execution
 unique_ptr<GlobalOperatorState> PhysicalTableInOutFunction::GetGlobalOperatorState(ClientContext &context) const {
 	auto result = make_unique<TableInOutGlobalState>();
 	if (function.init_global) {
-		TableFunctionInitInput input(bind_data.get(), column_ids, nullptr);
+		TableFunctionInitInput input(bind_data.get(), column_ids, vector<idx_t>(), nullptr);
 		result->global_state = function.init_global(context, input);
 	}
 	return move(result);
@@ -79299,11 +79304,22 @@ PhysicalTableScan::PhysicalTableScan(vector<LogicalType> types, TableFunction fu
       table_filters(move(table_filters_p)) {
 }
 
+PhysicalTableScan::PhysicalTableScan(vector<LogicalType> types, TableFunction function_p,
+                                     unique_ptr<FunctionData> bind_data_p, vector<LogicalType> returned_types_p,
+                                     vector<column_t> column_ids_p, vector<idx_t> projection_ids_p,
+                                     vector<string> names_p, unique_ptr<TableFilterSet> table_filters_p,
+                                     idx_t estimated_cardinality)
+    : PhysicalOperator(PhysicalOperatorType::TABLE_SCAN, move(types), estimated_cardinality),
+      function(move(function_p)), bind_data(move(bind_data_p)), returned_types(move(returned_types_p)),
+      column_ids(move(column_ids_p)), projection_ids(move(projection_ids_p)), names(move(names_p)),
+      table_filters(move(table_filters_p)) {
+}
+
 class TableScanGlobalSourceState : public GlobalSourceState {
 public:
 	TableScanGlobalSourceState(ClientContext &context, const PhysicalTableScan &op) {
 		if (op.function.init_global) {
-			TableFunctionInitInput input(op.bind_data.get(), op.column_ids, op.table_filters.get());
+			TableFunctionInitInput input(op.bind_data.get(), op.column_ids, op.projection_ids, op.table_filters.get());
 			global_state = op.function.init_global(context, input);
 			if (global_state) {
 				max_threads = global_state->MaxThreads();
@@ -79326,7 +79342,7 @@ public:
 	TableScanLocalSourceState(ExecutionContext &context, TableScanGlobalSourceState &gstate,
 	                          const PhysicalTableScan &op) {
 		if (op.function.init_local) {
-			TableFunctionInitInput input(op.bind_data.get(), op.column_ids, op.table_filters.get());
+			TableFunctionInitInput input(op.bind_data.get(), op.column_ids, op.projection_ids, op.table_filters.get());
 			local_state = op.function.init_local(context, input, gstate.global_state.get());
 		}
 	}
@@ -79383,12 +79399,13 @@ string PhysicalTableScan::ParamsToString() const {
 		result += "\n[INFOSEPARATOR]\n";
 	}
 	if (function.projection_pushdown) {
-		for (idx_t i = 0; i < column_ids.size(); i++) {
-			if (column_ids[i] < names.size()) {
+		for (idx_t i = 0; i < projection_ids.size(); i++) {
+			const auto &column_id = column_ids[projection_ids[i]];
+			if (column_id < names.size()) {
 				if (i > 0) {
 					result += "\n";
 				}
-				result += names[column_ids[i]];
+				result += names[column_id];
 			}
 		}
 	}
@@ -83671,7 +83688,6 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalFilter &op
 
 
 
-
 namespace duckdb {
 
 unique_ptr<TableFilterSet> CreateTableFilterSet(TableFilterSet &table_filters, vector<column_t> &column_ids) {
@@ -83715,8 +83731,9 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalGet &op) {
 	// create the table scan node
 	if (!op.function.projection_pushdown) {
 		// function does not support projection pushdown
-		auto node = make_unique<PhysicalTableScan>(op.returned_types, op.function, move(op.bind_data), op.column_ids,
-		                                           op.names, move(table_filters), op.estimated_cardinality);
+		auto node = make_unique<PhysicalTableScan>(op.returned_types, op.function, move(op.bind_data),
+		                                           op.returned_types, op.column_ids, vector<column_t>(), op.names,
+		                                           move(table_filters), op.estimated_cardinality);
 		// first check if an additional projection is necessary
 		if (op.column_ids.size() == op.returned_types.size()) {
 			bool projection_necessary = false;
@@ -83751,8 +83768,9 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalGet &op) {
 		projection->children.push_back(move(node));
 		return move(projection);
 	} else {
-		return make_unique<PhysicalTableScan>(op.types, op.function, move(op.bind_data), op.column_ids, op.names,
-		                                      move(table_filters), op.estimated_cardinality);
+		return make_unique<PhysicalTableScan>(op.types, op.function, move(op.bind_data), op.returned_types,
+		                                      op.column_ids, op.projection_ids, op.names, move(table_filters),
+		                                      op.estimated_cardinality);
 	}
 }
 
@@ -116769,6 +116787,7 @@ string ScalarMacroFunction::ToSQL(const string &schema, const string &name) {
 
 
 
+
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -116852,6 +116871,7 @@ struct ArrowScanFunctionData : public PyTableFunctionData {
 	}
 	//! This holds the original list type (col_idx, [ArrowListType,size])
 	unordered_map<idx_t, unique_ptr<ArrowConvertData>> arrow_convert_data;
+	vector<LogicalType> all_types;
 	atomic<idx_t> lines_read;
 	ArrowSchemaWrapper schema_root;
 	idx_t rows_per_thread;
@@ -116872,6 +116892,8 @@ struct ArrowScanLocalState : public LocalTableFunctionState {
 	//! Store child vectors for Arrow Dictionary Vectors (col-idx,vector)
 	unordered_map<idx_t, unique_ptr<Vector>> arrow_dictionary_vectors;
 	TableFilterSet *filters = nullptr;
+	//! The DataChunk containing all read columns (even filter columns that are immediately removed)
+	DataChunk all_columns;
 };
 
 struct ArrowScanGlobalState : public GlobalTableFunctionState {
@@ -116880,9 +116902,16 @@ struct ArrowScanGlobalState : public GlobalTableFunctionState {
 	idx_t max_threads = 1;
 	bool done = false;
 
+	vector<idx_t> projection_ids;
+	vector<LogicalType> scanned_types;
+
 	idx_t MaxThreads() const override {
 		return max_threads;
 	}
+
+	bool CanRemoveFilterColumns() const {
+		return !projection_ids.empty();
+	}
 };
 
 struct ArrowTableFunction {
@@ -116927,7 +116956,6 @@ private:
 
 
 
-
 namespace duckdb {
 
 LogicalType GetArrowLogicalType(ArrowSchema &schema,
@@ -117139,6 +117167,7 @@ unique_ptr<FunctionData> ArrowTableFunction::ArrowScanBind(ClientContext &contex
 		names.push_back(name);
 	}
 	RenameArrowColumns(names);
+	res->all_types = return_types;
 	return move(res);
 }
 
@@ -117190,6 +117219,16 @@ unique_ptr<GlobalTableFunctionState> ArrowTableFunction::ArrowScanInitGlobal(Cli
 	auto result = make_unique<ArrowScanGlobalState>();
 	result->stream = ProduceArrowScan(bind_data, input.column_ids, input.filters);
 	result->max_threads = ArrowScanMaxThreads(context, input.bind_data);
+	if (input.CanRemoveFilterColumns()) {
+		result->projection_ids = input.projection_ids;
+		for (const auto &col_idx : input.column_ids) {
+			if (col_idx == COLUMN_IDENTIFIER_ROW_ID) {
+				result->scanned_types.emplace_back(LogicalType::ROW_TYPE);
+			} else {
+				result->scanned_types.push_back(bind_data.all_types[col_idx]);
+			}
+		}
+	}
 	return move(result);
 }
 
@@ -117201,6 +117240,10 @@ unique_ptr<LocalTableFunctionState> ArrowTableFunction::ArrowScanInitLocal(Execu
 	auto result = make_unique<ArrowScanLocalState>(move(current_chunk));
 	result->column_ids = input.column_ids;
 	result->filters = input.filters;
+	if (input.CanRemoveFilterColumns()) {
+		auto &asgs = (ArrowScanGlobalState &)*global_state_p;
+		result->all_columns.Initialize(context.client, asgs.scanned_types);
+	}
 	if (!ArrowScanParallelStateNext(context.client, input.bind_data, *result, global_state)) {
 		return nullptr;
 	}
@@ -117223,8 +117266,16 @@ void ArrowTableFunction::ArrowScanFunction(ClientContext &context, TableFunction
 	}
 	int64_t output_size = MinValue<int64_t>(STANDARD_VECTOR_SIZE, state.chunk->arrow_array.length - state.chunk_offset);
 	data.lines_read += output_size;
-	output.SetCardinality(output_size);
-	ArrowToDuckDB(state, data.arrow_convert_data, output, data.lines_read - output_size);
+	if (global_state.CanRemoveFilterColumns()) {
+		state.all_columns.Reset();
+		state.all_columns.SetCardinality(output_size);
+		ArrowToDuckDB(state, data.arrow_convert_data, state.all_columns, data.lines_read - output_size);
+		output.ReferenceColumns(state.all_columns, global_state.projection_ids);
+	} else {
+		output.SetCardinality(output_size);
+		ArrowToDuckDB(state, data.arrow_convert_data, output, data.lines_read - output_size);
+	}
+
 	output.Verify();
 	state.chunk_offset += output.size();
 }
@@ -117239,6 +117290,7 @@ void ArrowTableFunction::RegisterFunction(BuiltinFunctions &set) {
 	arrow.cardinality = ArrowScanCardinality;
 	arrow.projection_pushdown = true;
 	arrow.filter_pushdown = true;
+	arrow.filter_prune = true;
 	set.AddFunction(arrow);
 }
 
@@ -123436,10 +123488,6 @@ void BuiltinFunctions::RegisterSQLiteFunctions() {
 
 
 
-
-
-
-
 namespace duckdb {
 
 //===--------------------------------------------------------------------===//
@@ -123451,6 +123499,8 @@ bool TableScanParallelStateNext(ClientContext &context, const FunctionData *bind
 struct TableScanLocalState : public LocalTableFunctionState {
 	//! The current position in the scan
 	TableScanState scan_state;
+	//! The DataChunk containing all read columns (even filter columns that are immediately removed)
+	DataChunk all_columns;
 };
 
 static storage_t GetStorageIndex(TableCatalogEntry &table, column_t column_id) {
@@ -123472,9 +123522,16 @@ struct TableScanGlobalState : public GlobalTableFunctionState {
 	mutex lock;
 	idx_t max_threads;
 
+	vector<idx_t> projection_ids;
+	vector<LogicalType> scanned_types;
+
 	idx_t MaxThreads() const override {
 		return max_threads;
 	}
+
+	bool CanRemoveFilterColumns() const {
+		return !projection_ids.empty();
+	}
 };
 
 static unique_ptr<LocalTableFunctionState> TableScanInitLocal(ExecutionContext &context, TableFunctionInitInput &input,
@@ -123488,6 +123545,10 @@ static unique_ptr<LocalTableFunctionState> TableScanInitLocal(ExecutionContext &
 	}
 	result->scan_state.Initialize(move(column_ids), input.filters);
 	TableScanParallelStateNext(context.client, input.bind_data, result.get(), gstate);
+	if (input.CanRemoveFilterColumns()) {
+		auto &tsgs = (TableScanGlobalState &)*gstate;
+		result->all_columns.Initialize(context.client, tsgs.scanned_types);
+	}
 	return move(result);
 }
 
@@ -123496,8 +123557,18 @@ unique_ptr<GlobalTableFunctionState> TableScanInitGlobal(ClientContext &context,
 	D_ASSERT(input.bind_data);
 	auto &bind_data = (const TableScanBindData &)*input.bind_data;
 	auto result = make_unique<TableScanGlobalState>(context, input.bind_data);
-
 	bind_data.table->storage->InitializeParallelScan(context, result->state);
+	if (input.CanRemoveFilterColumns()) {
+		result->projection_ids = input.projection_ids;
+		const auto &columns = bind_data.table->columns;
+		for (const auto &col_idx : input.column_ids) {
+			if (col_idx == COLUMN_IDENTIFIER_ROW_ID) {
+				result->scanned_types.emplace_back(LogicalType::ROW_TYPE);
+			} else {
+				result->scanned_types.push_back(columns[col_idx].Type());
+			}
+		}
+	}
 	return move(result);
 }
 
@@ -123514,12 +123585,17 @@ static unique_ptr<BaseStatistics> TableScanStatistics(ClientContext &context, co
 
 static void TableScanFunc(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) {
 	auto &bind_data = (TableScanBindData &)*data_p.bind_data;
+	auto &gstate = (TableScanGlobalState &)*data_p.global_state;
 	auto &state = (TableScanLocalState &)*data_p.local_state;
 	auto &transaction = Transaction::GetTransaction(context);
 	do {
 		if (bind_data.is_create_index) {
 			bind_data.table->storage->CreateIndexScan(
 			    state.scan_state, output, TableScanType::TABLE_SCAN_COMMITTED_ROWS_OMIT_PERMANENTLY_DELETED);
+		} else if (gstate.CanRemoveFilterColumns()) {
+			state.all_columns.Reset();
+			bind_data.table->storage->Scan(transaction, state.all_columns, state.scan_state);
+			output.ReferenceColumns(state.all_columns, gstate.projection_ids);
 		} else {
 			bind_data.table->storage->Scan(transaction, output, state.scan_state);
 		}
@@ -123851,6 +123927,7 @@ TableFunction TableScanFunction::GetFunction() {
 	scan_function.get_batch_index = TableScanGetBatchIndex;
 	scan_function.projection_pushdown = true;
 	scan_function.filter_pushdown = true;
+	scan_function.filter_prune = true;
 	scan_function.serialize = TableScanSerialize;
 	scan_function.deserialize = TableScanDeserialize;
 	return scan_function;
@@ -124067,7 +124144,7 @@ TableFunction::TableFunction(string name, vector<LogicalType> arguments, table_f
       init_local(init_local), function(function), in_out_function(nullptr), statistics(nullptr), dependency(nullptr),
       cardinality(nullptr), pushdown_complex_filter(nullptr), to_string(nullptr), table_scan_progress(nullptr),
       get_batch_index(nullptr), serialize(nullptr), deserialize(nullptr), projection_pushdown(false),
-      filter_pushdown(false) {
+      filter_pushdown(false), filter_prune(false) {
 }
 
 TableFunction::TableFunction(const vector<LogicalType> &arguments, table_function_t function,
@@ -124079,7 +124156,8 @@ TableFunction::TableFunction()
     : SimpleNamedParameterFunction("", {}), bind(nullptr), init_global(nullptr), init_local(nullptr), function(nullptr),
       in_out_function(nullptr), statistics(nullptr), dependency(nullptr), cardinality(nullptr),
       pushdown_complex_filter(nullptr), to_string(nullptr), table_scan_progress(nullptr), get_batch_index(nullptr),
-      serialize(nullptr), deserialize(nullptr), projection_pushdown(false), filter_pushdown(false) {
+      serialize(nullptr), deserialize(nullptr), projection_pushdown(false), filter_pushdown(false),
+      filter_prune(false) {
 }
 
 } // namespace duckdb
@@ -149270,7 +149348,7 @@ private:
 
 private:
 	template <class T>
-	void ClearUnusedExpressions(vector<T> &list, idx_t table_idx);
+	void ClearUnusedExpressions(vector<T> &list, idx_t table_idx, bool replace = true);
 
 	//! Perform a replacement of the ColumnBinding, iterating over all the currently found column references and
 	//! replacing the bindings
@@ -150793,7 +150871,7 @@ void RemoveUnusedColumns::ReplaceBinding(ColumnBinding current_binding, ColumnBi
 }
 
 template <class T>
-void RemoveUnusedColumns::ClearUnusedExpressions(vector<T> &list, idx_t table_idx) {
+void RemoveUnusedColumns::ClearUnusedExpressions(vector<T> &list, idx_t table_idx, bool replace) {
 	idx_t offset = 0;
 	for (idx_t col_idx = 0; col_idx < list.size(); col_idx++) {
 		auto current_binding = ColumnBinding(table_idx, col_idx + offset);
@@ -150803,7 +150881,7 @@ void RemoveUnusedColumns::ClearUnusedExpressions(vector<T> &list, idx_t table_id
 			list.erase(list.begin() + col_idx);
 			offset++;
 			col_idx--;
-		} else if (offset > 0) {
+		} else if (offset > 0 && replace) {
 			// column is used but the ColumnBinding has changed because of removed columns
 			ReplaceBinding(current_binding, ColumnBinding(table_idx, col_idx));
 		}
@@ -150976,6 +151054,17 @@ void RemoveUnusedColumns::VisitOperator(LogicalOperator &op) {
 		LogicalOperatorVisitor::VisitOperatorExpressions(op);
 		if (!everything_referenced) {
 			auto &get = (LogicalGet &)op;
+
+			// Create "selection vector" of all column ids
+			vector<idx_t> proj_sel;
+			for (idx_t col_idx = 0; col_idx < get.column_ids.size(); col_idx++) {
+				proj_sel.push_back(col_idx);
+			}
+			// Create a copy that we can use to match ids later
+			auto col_sel = proj_sel;
+			// Clear unused ids, exclude filter columns that are projected out immediately
+			ClearUnusedExpressions(proj_sel, get.table_index, false);
+
 			// for every table filter, push a column binding into the column references map to prevent the column from
 			// being projected out
 			for (auto &filter : get.table_filters.filters) {
@@ -150994,8 +151083,31 @@ void RemoveUnusedColumns::VisitOperator(LogicalOperator &op) {
 					column_references.insert(make_pair(filter_binding, vector<BoundColumnRefExpression *>()));
 				}
 			}
-			// table scan: figure out which columns are referenced
-			ClearUnusedExpressions(get.column_ids, get.table_index);
+
+			// Clear unused ids, include filter columns that are projected out immediately
+			ClearUnusedExpressions(col_sel, get.table_index);
+
+			// Now set the column ids in the LogicalGet using the "selection vector"
+			vector<column_t> column_ids;
+			column_ids.reserve(col_sel.size());
+			for (auto col_sel_idx : col_sel) {
+				column_ids.push_back(get.column_ids[col_sel_idx]);
+			}
+			get.column_ids = move(column_ids);
+
+			if (get.function.filter_prune) {
+				// Now set the projection cols by matching the "selection vector" that excludes filter columns
+				// with the "selection vector" that includes filter columns
+				idx_t col_idx = 0;
+				for (auto proj_sel_idx : proj_sel) {
+					for (; col_idx < col_sel.size(); col_idx++) {
+						if (proj_sel_idx == col_sel[col_idx]) {
+							get.projection_ids.push_back(col_idx);
+							break;
+						}
+					}
+				}
+			}
 
 			if (get.column_ids.empty()) {
 				// this generally means we are only interested in whether or not anything exists in the table (e.g.
@@ -190301,7 +190413,6 @@ unique_ptr<LogicalOperator> LogicalFilter::Deserialize(LogicalDeserializationSta
 
 
 
-
 namespace duckdb {
 
 LogicalGet::LogicalGet(idx_t table_index, TableFunction function, unique_ptr<FunctionData> bind_data,
@@ -190339,8 +190450,14 @@ vector<ColumnBinding> LogicalGet::GetColumnBindings() {
 		return {ColumnBinding(table_index, 0)};
 	}
 	vector<ColumnBinding> result;
-	for (idx_t i = 0; i < column_ids.size(); i++) {
-		result.emplace_back(table_index, i);
+	if (projection_ids.empty()) {
+		for (idx_t col_idx = 0; col_idx < column_ids.size(); col_idx++) {
+			result.emplace_back(table_index, col_idx);
+		}
+	} else {
+		for (auto proj_id : projection_ids) {
+			result.emplace_back(table_index, proj_id);
+		}
 	}
 	return result;
 }
@@ -190349,11 +190466,23 @@ void LogicalGet::ResolveTypes() {
 	if (column_ids.empty()) {
 		column_ids.push_back(COLUMN_IDENTIFIER_ROW_ID);
 	}
-	for (auto &index : column_ids) {
-		if (index == COLUMN_IDENTIFIER_ROW_ID) {
-			types.emplace_back(LogicalType::ROW_TYPE);
-		} else {
-			types.push_back(returned_types[index]);
+
+	if (projection_ids.empty()) {
+		for (auto &index : column_ids) {
+			if (index == COLUMN_IDENTIFIER_ROW_ID) {
+				types.emplace_back(LogicalType::ROW_TYPE);
+			} else {
+				types.push_back(returned_types[index]);
+			}
+		}
+	} else {
+		for (auto &proj_index : projection_ids) {
+			auto &index = column_ids[proj_index];
+			if (index == COLUMN_IDENTIFIER_ROW_ID) {
+				types.emplace_back(LogicalType::ROW_TYPE);
+			} else {
+				types.push_back(returned_types[index]);
+			}
 		}
 	}
 }
@@ -190373,6 +190502,7 @@ void LogicalGet::Serialize(FieldWriter &writer) const {
 	writer.WriteRegularSerializableList(returned_types);
 	writer.WriteList<string>(names);
 	writer.WriteList<column_t>(column_ids);
+	writer.WriteList<column_t>(projection_ids);
 	writer.WriteSerializable(table_filters);
 
 	FunctionSerializer::SerializeBase<TableFunction>(writer, function, bind_data.get());
@@ -190395,6 +190525,7 @@ unique_ptr<LogicalOperator> LogicalGet::Deserialize(LogicalDeserializationState
 	auto returned_types = reader.ReadRequiredSerializableList<LogicalType, LogicalType>();
 	auto returned_names = reader.ReadRequiredList<string>();
 	auto column_ids = reader.ReadRequiredList<column_t>();
+	auto projection_ids = reader.ReadRequiredList<column_t>();
 	auto table_filters = reader.ReadRequiredSerializable<TableFilterSet>();
 
 	unique_ptr<FunctionData> bind_data;
@@ -190439,6 +190570,7 @@ unique_ptr<LogicalOperator> LogicalGet::Deserialize(LogicalDeserializationState
 
 	auto result = make_unique<LogicalGet>(table_index, function, move(bind_data), returned_types, returned_names);
 	result->column_ids = move(column_ids);
+	result->projection_ids = move(projection_ids);
 	result->table_filters = move(*table_filters);
 	result->parameters = move(parameters);
 	result->named_parameters = move(named_parameters);

package/src/duckdb.hpp

@@ -11,8 +11,8 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 #pragma once
 #define DUCKDB_AMALGAMATION 1
 #define DUCKDB_AMALGAMATION_EXTENDED 1
-#define DUCKDB_SOURCE_ID "f21e66bf8"
-#define DUCKDB_VERSION "v0.5.2-dev494"
+#define DUCKDB_SOURCE_ID "9f84ca408"
+#define DUCKDB_VERSION "v0.5.2-dev512"
 //===----------------------------------------------------------------------===//
 //                         DuckDB
 //
@@ -4855,7 +4855,7 @@ public:
 	DUCKDB_API void Fuse(DataChunk &other);
 
 	//! Makes this DataChunk reference the specified columns in the other DataChunk
-	DUCKDB_API void ReferenceColumns(DataChunk &other, vector<column_t> column_ids);
+	DUCKDB_API void ReferenceColumns(DataChunk &other, const vector<column_t> &column_ids);
 
 	//! Turn all the vectors from the chunk into flat vectors
 	DUCKDB_API void Flatten();
@@ -12615,21 +12615,37 @@ struct TableFunctionBindInput {
 
 struct TableFunctionInitInput {
 	TableFunctionInitInput(const FunctionData *bind_data_p, const vector<column_t> &column_ids_p,
-	                       TableFilterSet *filters_p)
-	    : bind_data(bind_data_p), column_ids(column_ids_p), filters(filters_p) {
+	                       const vector<idx_t> &projection_ids_p, TableFilterSet *filters_p)
+	    : bind_data(bind_data_p), column_ids(column_ids_p), projection_ids(projection_ids_p), filters(filters_p) {
 	}
 
 	const FunctionData *bind_data;
 	const vector<column_t> &column_ids;
+	const vector<idx_t> projection_ids;
 	TableFilterSet *filters;
+
+	bool CanRemoveFilterColumns() const {
+		if (projection_ids.empty()) {
+			// Not set, can't remove filter columns
+			return false;
+		} else if (projection_ids.size() == column_ids.size()) {
+			// Filter column is used in remainder of plan, can't remove
+			return false;
+		} else {
+			// Less columns need to be projected out than that we scan
+			return true;
+		}
+	}
 };
 
 struct TableFunctionInput {
+public:
 	TableFunctionInput(const FunctionData *bind_data_p, LocalTableFunctionState *local_state_p,
 	                   GlobalTableFunctionState *global_state_p)
 	    : bind_data(bind_data_p), local_state(local_state_p), global_state(global_state_p) {
 	}
 
+public:
 	const FunctionData *bind_data;
 	LocalTableFunctionState *local_state;
 	GlobalTableFunctionState *global_state;
@@ -12722,6 +12738,9 @@ public:
 	//! Whether or not the table function supports filter pushdown. If not supported a filter will be added
 	//! that applies the table filter directly.
 	bool filter_pushdown;
+	//! Whether or not the table function can immediately prune out filter columns that are unused in the remainder of
+	//! the query plan, e.g., "SELECT i FROM tbl WHERE j = 42;" - j does not need to leave the table function at all
+	bool filter_prune;
 	//! Additional function info, passed to the bind
 	shared_ptr<TableFunctionInfo> function_info;
 };
@@ -28963,6 +28982,8 @@ public:
 	vector<string> names;
 	//! Bound column IDs
 	vector<column_t> column_ids;
+	//! Columns that are used outside of the scan
+	vector<idx_t> projection_ids;
 	//! Filters pushed down for table scan
 	TableFilterSet table_filters;
 	//! The set of input parameters for the table function