duckdb 0.8.2-dev33.0 → 0.8.2-dev3334.0

package/src/duckdb/src/function/table/read_csv.cpp

@@ -14,6 +14,10 @@
 #include "duckdb/common/multi_file_reader.hpp"
 #include "duckdb/main/client_data.hpp"
 #include "duckdb/execution/operator/persistent/csv_line_info.hpp"
+#include "duckdb/execution/operator/persistent/csv_rejects_table.hpp"
+#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
+
 #include <limits>
 
 namespace duckdb {
@@ -42,6 +46,43 @@ void ReadCSVData::FinalizeRead(ClientContext &context) {
 		// not supported for parallel CSV reading
 		single_threaded = true;
 	}
+
+	// Validate rejects_table options
+	if (!options.rejects_table_name.empty()) {
+		if (!options.ignore_errors) {
+			throw BinderException("REJECTS_TABLE option is only supported when IGNORE_ERRORS is set to true");
+		}
+		if (options.file_options.union_by_name) {
+			throw BinderException("REJECTS_TABLE option is not supported when UNION_BY_NAME is set to true");
+		}
+	}
+
+	if (!options.rejects_recovery_columns.empty()) {
+		if (options.rejects_table_name.empty()) {
+			throw BinderException(
+			    "REJECTS_RECOVERY_COLUMNS option is only supported when REJECTS_TABLE is set to a table name");
+		}
+		for (auto &recovery_col : options.rejects_recovery_columns) {
+			bool found = false;
+			for (idx_t col_idx = 0; col_idx < return_names.size(); col_idx++) {
+				if (StringUtil::CIEquals(return_names[col_idx], recovery_col)) {
+					options.rejects_recovery_column_ids.push_back(col_idx);
+					found = true;
+					break;
+				}
+			}
+			if (!found) {
+				throw BinderException("Unsupported parameter for REJECTS_RECOVERY_COLUMNS: column \"%s\" not found",
+				                      recovery_col);
+			}
+		}
+	}
+
+	if (options.rejects_limit != 0) {
+		if (options.rejects_table_name.empty()) {
+			throw BinderException("REJECTS_LIMIT option is only supported when REJECTS_TABLE is set to a table name");
+		}
+	}
 }
 
 uint8_t GetCandidateSpecificity(const LogicalType &candidate_type) {
@@ -65,13 +106,14 @@ uint8_t GetCandidateSpecificity(const LogicalType &candidate_type) {
 
 static unique_ptr<FunctionData> ReadCSVBind(ClientContext &context, TableFunctionBindInput &input,
                                             vector<LogicalType> &return_types, vector<string> &names) {
+
 	auto result = make_uniq<ReadCSVData>();
 	auto &options = result->options;
 	result->files = MultiFileReader::GetFileList(context, input.inputs[0], "CSV");
 
 	bool explicitly_set_columns = false;
 	for (auto &kv : input.named_parameters) {
-		if (MultiFileReader::ParseOption(kv.first, kv.second, options.file_options)) {
+		if (MultiFileReader::ParseOption(kv.first, kv.second, options.file_options, context)) {
 			continue;
 		}
 		auto loption = StringUtil::Lower(kv.first);
@@ -180,9 +222,7 @@ static unique_ptr<FunctionData> ReadCSVBind(ClientContext &context, TableFunctio
 			options.SetReadOption(loption, kv.second, names);
 		}
 	}
-	if (options.file_options.auto_detect_hive_partitioning) {
-		options.file_options.hive_partitioning = MultiFileReaderOptions::AutoDetectHivePartitioning(result->files);
-	}
+	options.file_options.AutoDetectHivePartitioning(result->files, context);
 
 	if (!options.auto_detect && return_types.empty()) {
 		throw BinderException("read_csv requires columns to be specified through the 'columns' option. Use "
@@ -235,9 +275,15 @@ static unique_ptr<FunctionData> ReadCSVBind(ClientContext &context, TableFunctio
 	} else {
 		result->reader_bind = MultiFileReader::BindOptions(options.file_options, result->files, return_types, names);
 	}
+
 	result->return_types = return_types;
 	result->return_names = names;
 	result->FinalizeRead(context);
+
+	if (options.auto_detect) {
+		result->initial_reader->options = options;
+	}
+
 	return std::move(result);
 }
 
@@ -261,10 +307,6 @@ public:
 	      line_info(main_mutex, batch_to_tuple_end, tuple_start, tuple_end) {
 		file_handle->DisableReset();
 		current_file_path = files_path_p[0];
-		line_info.lines_read[0] = rows_to_skip;
-		if (has_header) {
-			line_info.lines_read[0]++;
-		}
 		file_size = file_handle->FileSize();
 		first_file_size = file_size;
 		on_disk_file = file_handle->OnDiskFile();
@@ -291,12 +333,20 @@ public:
 		// Initialize all the book-keeping variables
 		auto file_count = files_path_p.size();
 		line_info.current_batches.resize(file_count);
+		line_info.lines_read.resize(file_count);
 		tuple_start.resize(file_count);
 		tuple_end.resize(file_count);
 		tuple_end_to_batch.resize(file_count);
 		batch_to_tuple_end.resize(file_count);
+
+		// Initialize the lines read
+		line_info.lines_read[0][0] = rows_to_skip;
+		if (has_header) {
+			line_info.lines_read[0][0]++;
+		}
 	}
-	ParallelCSVGlobalState() : line_info(main_mutex, batch_to_tuple_end, tuple_start, tuple_end) {
+	explicit ParallelCSVGlobalState(idx_t system_threads_p)
+	    : system_threads(system_threads_p), line_info(main_mutex, batch_to_tuple_end, tuple_start, tuple_end) {
 		running_threads = MaxThreads();
 	}
 
@@ -358,7 +408,7 @@ private:
 	//! How many bytes we should execute per local state
 	idx_t bytes_per_local_state;
 	//! Size of first file
-	idx_t first_file_size;
+	idx_t first_file_size = 0;
 	//! Whether or not this is an on-disk file
 	bool on_disk_file = true;
 	//! Basically max number of threads in DuckDB
@@ -501,6 +551,9 @@ bool ParallelCSVGlobalState::Next(ClientContext &context, const ReadCSVData &bin
 			current_csv_position = 0;
 			file_number++;
 			local_batch_index = 0;
+
+			line_info.lines_read[file_number][local_batch_index] = (bind_data.options.has_header ? 1 : 0);
+
 			current_buffer =
 			    make_shared<CSVBuffer>(context, buffer_size, *file_handle, current_csv_position, file_number);
 			next_buffer = shared_ptr<CSVBuffer>(
@@ -512,7 +565,7 @@ bool ParallelCSVGlobalState::Next(ClientContext &context, const ReadCSVData &bin
 		}
 	}
 	// set up the current buffer
-	line_info.current_batches.back().insert(local_batch_index);
+	line_info.current_batches[file_number].insert(local_batch_index);
 	auto result = make_uniq<CSVBufferRead>(current_buffer, next_buffer, next_byte, next_byte + bytes_per_local_state,
 	                                       batch_index++, local_batch_index++, &line_info);
 	// move the byte index of the CSV reader to the next buffer
@@ -551,11 +604,12 @@ bool ParallelCSVGlobalState::Next(ClientContext &context, const ReadCSVData &bin
 		reader->options.file_path = current_file_path;
 		MultiFileReader::InitializeReader(*reader, bind_data.options.file_options, bind_data.reader_bind,
 		                                  bind_data.return_types, bind_data.return_names, column_ids, nullptr,
-		                                  bind_data.files.front());
+		                                  bind_data.files.front(), context);
 	} else {
 		// update the current reader
 		reader->SetBufferRead(std::move(result));
 	}
+
 	return true;
 }
 void ParallelCSVGlobalState::UpdateVerification(VerificationPositions positions, idx_t file_number_p, idx_t batch_idx) {
@@ -574,7 +628,7 @@ void ParallelCSVGlobalState::UpdateLinesRead(CSVBufferRead &buffer_read, idx_t f
 	auto lines_read = buffer_read.lines_read;
 	lock_guard<mutex> parallel_lock(main_mutex);
 	line_info.current_batches[file_idx].erase(batch_idx);
-	line_info.lines_read[batch_idx] += lines_read;
+	line_info.lines_read[file_idx][batch_idx] += lines_read;
 }
 
 bool LineInfo::CanItGetLine(idx_t file_idx, idx_t batch_idx) {
@@ -593,21 +647,36 @@ bool LineInfo::CanItGetLine(idx_t file_idx, idx_t batch_idx) {
 }
 
 // Returns the 1-indexed line number
-idx_t LineInfo::GetLine(idx_t batch_idx, idx_t line_error, idx_t file_idx, idx_t cur_start, bool verify) {
+idx_t LineInfo::GetLine(idx_t batch_idx, idx_t line_error, idx_t file_idx, idx_t cur_start, bool verify,
+                        bool stop_at_first) {
 	unique_ptr<lock_guard<mutex>> parallel_lock;
 	if (!verify) {
 		parallel_lock = duckdb::make_uniq<lock_guard<mutex>>(main_mutex);
 	}
 	idx_t line_count = 0;
+
+	if (!stop_at_first) {
+		// Figure out the amount of lines read in the current file
+		auto &file_batches = current_batches[file_idx];
+		for (auto &batch : file_batches) {
+			if (batch > batch_idx) {
+				break;
+			}
+			line_count += lines_read[file_idx][batch];
+		}
+		return line_count + line_error + 1;
+	}
+
+	// Otherwise, check if we already have an error on another thread
 	if (done) {
 		// line count is 0-indexed, but we want to return 1-indexed
 		return first_line + 1;
 	}
 	for (idx_t i = 0; i <= batch_idx; i++) {
-		if (lines_read.find(i) == lines_read.end() && i != batch_idx) {
+		if (lines_read[file_idx].find(i) == lines_read[file_idx].end() && i != batch_idx) {
 			throw InternalException("Missing batch index on Parallel CSV Reader GetLine");
 		}
-		line_count += lines_read[i];
+		line_count += lines_read[file_idx][i];
 	}
 
 	// before we are done, if this is not a call in Verify() we must check Verify up to this batch
@@ -625,7 +694,7 @@ static unique_ptr<GlobalTableFunctionState> ParallelCSVInitGlobal(ClientContext
 	auto &bind_data = input.bind_data->CastNoConst<ReadCSVData>();
 	if (bind_data.files.empty()) {
 		// This can happen when a filename based filter pushdown has eliminated all possible files for this scan.
-		return make_uniq<ParallelCSVGlobalState>();
+		return make_uniq<ParallelCSVGlobalState>(context.db->NumberOfThreads());
 	}
 	unique_ptr<CSVFileHandle> file_handle;
 
@@ -782,7 +851,7 @@ private:
 			}
 			MultiFileReader::InitializeReader(*result, bind_data.options.file_options, bind_data.reader_bind,
 			                                  bind_data.return_types, bind_data.return_names, column_ids, nullptr,
-			                                  bind_data.files.front());
+			                                  bind_data.files.front(), context);
 		}
 		total_size = result->file_handle->FileSize();
 		return result;
@@ -831,14 +900,14 @@ static unique_ptr<GlobalTableFunctionState> SingleThreadedCSVInit(ClientContext
 	}
 	MultiFileReader::InitializeReader(*result->initial_reader, bind_data.options.file_options, bind_data.reader_bind,
 	                                  bind_data.return_types, bind_data.return_names, input.column_ids, input.filters,
-	                                  bind_data.files.front());
+	                                  bind_data.files.front(), context);
 	for (auto &reader : bind_data.union_readers) {
 		if (!reader) {
 			continue;
 		}
 		MultiFileReader::InitializeReader(*reader, bind_data.options.file_options, bind_data.reader_bind,
 		                                  bind_data.return_types, bind_data.return_names, input.column_ids,
-		                                  input.filters, bind_data.files.front());
+		                                  input.filters, bind_data.files.front(), context);
 	}
 	result->column_ids = input.column_ids;
 
@@ -914,6 +983,12 @@ static void SingleThreadedCSVFunction(ClientContext &context, TableFunctionInput
 //===--------------------------------------------------------------------===//
 static unique_ptr<GlobalTableFunctionState> ReadCSVInitGlobal(ClientContext &context, TableFunctionInitInput &input) {
 	auto &bind_data = input.bind_data->Cast<ReadCSVData>();
+
+	// Create the temporary rejects table
+	auto rejects_table = bind_data.options.rejects_table_name;
+	if (!rejects_table.empty()) {
+		CSVRejectsTable::GetOrCreate(context, rejects_table)->InitializeTable(context, bind_data);
+	}
 	if (bind_data.single_threaded) {
 		return SingleThreadedCSVInit(context, input);
 	} else {
@@ -974,6 +1049,9 @@ static void ReadCSVAddNamedParameters(TableFunction &table_function) {
 	table_function.named_parameters["max_line_size"] = LogicalType::VARCHAR;
 	table_function.named_parameters["maximum_line_size"] = LogicalType::VARCHAR;
 	table_function.named_parameters["ignore_errors"] = LogicalType::BOOLEAN;
+	table_function.named_parameters["rejects_table"] = LogicalType::VARCHAR;
+	table_function.named_parameters["rejects_limit"] = LogicalType::BIGINT;
+	table_function.named_parameters["rejects_recovery_columns"] = LogicalType::LIST(LogicalType::VARCHAR);
 	table_function.named_parameters["buffer_size"] = LogicalType::UBIGINT;
 	table_function.named_parameters["decimal_separator"] = LogicalType::VARCHAR;
 	table_function.named_parameters["parallel"] = LogicalType::BOOLEAN;
@@ -1062,6 +1140,10 @@ void BufferedCSVReaderOptions::Serialize(FieldWriter &writer) const {
 		csv_formats.push_back(format.second.format_specifier);
 	}
 	writer.WriteList<string>(csv_formats);
+	writer.WriteString(rejects_table_name);
+	writer.WriteField<idx_t>(rejects_limit);
+	writer.WriteList<string>(rejects_recovery_columns);
+	writer.WriteList<idx_t>(rejects_recovery_column_ids);
 }
 
 void BufferedCSVReaderOptions::Deserialize(FieldReader &reader) {
@@ -1109,6 +1191,10 @@ void BufferedCSVReaderOptions::Deserialize(FieldReader &reader) {
 		has_format[type] = true;
 		StrTimeFormat::ParseFormatSpecifier(format, date_format[type]);
 	}
+	rejects_table_name = reader.ReadRequired<string>();
+	rejects_limit = reader.ReadRequired<idx_t>();
+	rejects_recovery_columns = reader.ReadRequiredList<string>();
+	rejects_recovery_column_ids = reader.ReadRequiredList<idx_t>();
 }
 
 static void CSVReaderSerialize(FieldWriter &writer, const FunctionData *bind_data_p, const TableFunction &function) {
@@ -1151,6 +1237,20 @@ static unique_ptr<FunctionData> CSVReaderDeserialize(PlanDeserializationState &s
 	return std::move(result_data);
 }
 
+static void CSVReaderFormatSerialize(FormatSerializer &serializer, const optional_ptr<FunctionData> bind_data_p,
+                                     const TableFunction &function) {
+	auto &bind_data = bind_data_p->Cast<ReadCSVData>();
+	serializer.WriteProperty(100, "extra_info", function.extra_info);
+	serializer.WriteProperty(101, "csv_data", bind_data);
+}
+
+static unique_ptr<FunctionData> CSVReaderFormatDeserialize(FormatDeserializer &deserializer, TableFunction &function) {
+	unique_ptr<ReadCSVData> result;
+	deserializer.ReadProperty(100, "extra_info", function.extra_info);
+	deserializer.ReadProperty(101, "csv_data", result);
+	return std::move(result);
+}
+
 TableFunction ReadCSVTableFunction::GetFunction() {
 	TableFunction read_csv("read_csv", {LogicalType::VARCHAR}, ReadCSVFunction, ReadCSVBind, ReadCSVInitGlobal,
 	                       ReadCSVInitLocal);
@@ -1158,6 +1258,8 @@ TableFunction ReadCSVTableFunction::GetFunction() {
 	read_csv.pushdown_complex_filter = CSVComplexFilterPushdown;
 	read_csv.serialize = CSVReaderSerialize;
 	read_csv.deserialize = CSVReaderDeserialize;
+	read_csv.format_serialize = CSVReaderFormatSerialize;
+	read_csv.format_deserialize = CSVReaderFormatDeserialize;
 	read_csv.get_batch_index = CSVReaderGetBatchIndex;
 	read_csv.cardinality = CSVReaderCardinality;
 	read_csv.projection_pushdown = true;
@@ -1195,7 +1297,8 @@ unique_ptr<TableRef> ReadCSVReplacement(ClientContext &context, const string &ta
 	table_function->function = make_uniq<FunctionExpression>("read_csv_auto", std::move(children));
 
 	if (!FileSystem::HasGlob(table_name)) {
-		table_function->alias = FileSystem::ExtractBaseName(table_name);
+		auto &fs = FileSystem::GetFileSystem(context);
+		table_function->alias = fs.ExtractBaseName(table_name);
 	}
 
 	return std::move(table_function);

package/src/duckdb/src/function/table/system/test_all_types.cpp

@@ -17,7 +17,7 @@ struct TestAllTypesData : public GlobalTableFunctionState {
 	idx_t offset;
 };
 
-vector<TestType> TestAllTypesFun::GetTestTypes() {
+vector<TestType> TestAllTypesFun::GetTestTypes(bool use_large_enum) {
 	vector<TestType> result;
 	// scalar types/numerics
 	result.emplace_back(LogicalType::BOOLEAN, "bool");
@@ -70,22 +70,30 @@ vector<TestType> TestAllTypesFun::GetTestTypes() {
 	auto small_enum_ptr = FlatVector::GetData<string_t>(small_enum);
 	small_enum_ptr[0] = StringVector::AddStringOrBlob(small_enum, "DUCK_DUCK_ENUM");
 	small_enum_ptr[1] = StringVector::AddStringOrBlob(small_enum, "GOOSE");
-	result.emplace_back(LogicalType::ENUM("small_enum", small_enum, 2), "small_enum");
+	result.emplace_back(LogicalType::ENUM(small_enum, 2), "small_enum");
 
 	Vector medium_enum(LogicalType::VARCHAR, 300);
 	auto medium_enum_ptr = FlatVector::GetData<string_t>(medium_enum);
 	for (idx_t i = 0; i < 300; i++) {
 		medium_enum_ptr[i] = StringVector::AddStringOrBlob(medium_enum, string("enum_") + to_string(i));
 	}
-	result.emplace_back(LogicalType::ENUM("medium_enum", medium_enum, 300), "medium_enum");
-
-	// this is a big one... not sure if we should push this one here, but it's required for completeness
-	Vector large_enum(LogicalType::VARCHAR, 70000);
-	auto large_enum_ptr = FlatVector::GetData<string_t>(large_enum);
-	for (idx_t i = 0; i < 70000; i++) {
-		large_enum_ptr[i] = StringVector::AddStringOrBlob(large_enum, string("enum_") + to_string(i));
+	result.emplace_back(LogicalType::ENUM(medium_enum, 300), "medium_enum");
+
+	if (use_large_enum) {
+		// this is a big one... not sure if we should push this one here, but it's required for completeness
+		Vector large_enum(LogicalType::VARCHAR, 70000);
+		auto large_enum_ptr = FlatVector::GetData<string_t>(large_enum);
+		for (idx_t i = 0; i < 70000; i++) {
+			large_enum_ptr[i] = StringVector::AddStringOrBlob(large_enum, string("enum_") + to_string(i));
+		}
+		result.emplace_back(LogicalType::ENUM(large_enum, 70000), "large_enum");
+	} else {
+		Vector large_enum(LogicalType::VARCHAR, 2);
+		auto large_enum_ptr = FlatVector::GetData<string_t>(large_enum);
+		large_enum_ptr[0] = StringVector::AddStringOrBlob(large_enum, string("enum_") + to_string(0));
+		large_enum_ptr[1] = StringVector::AddStringOrBlob(large_enum, string("enum_") + to_string(69999));
+		result.emplace_back(LogicalType::ENUM(large_enum, 2), "large_enum");
 	}
-	result.emplace_back(LogicalType::ENUM("large_enum", large_enum, 70000), "large_enum");
 
 	// arrays
 	auto int_list_type = LogicalType::LIST(LogicalType::INTEGER);
@@ -196,29 +204,46 @@ vector<TestType> TestAllTypesFun::GetTestTypes() {
 	auto max_map_value = Value::MAP(ListType::GetChildType(map_type), map_values);
 	result.emplace_back(map_type, "map", std::move(min_map_value), std::move(max_map_value));
 
+	// union
+	child_list_t<LogicalType> members = {{"name", LogicalType::VARCHAR}, {"age", LogicalType::SMALLINT}};
+	auto union_type = LogicalType::UNION(members);
+	const Value &min = Value::UNION(members, 0, Value("Frank"));
+	const Value &max = Value::UNION(members, 1, Value::SMALLINT(5));
+	result.emplace_back(union_type, "union", min, max);
+
 	return result;
 }
 
+struct TestAllTypesBindData : public TableFunctionData {
+	vector<TestType> test_types;
+};
+
 static unique_ptr<FunctionData> TestAllTypesBind(ClientContext &context, TableFunctionBindInput &input,
                                                  vector<LogicalType> &return_types, vector<string> &names) {
-	auto test_types = TestAllTypesFun::GetTestTypes();
-	for (auto &test_type : test_types) {
-		return_types.push_back(std::move(test_type.type));
-		names.push_back(std::move(test_type.name));
+	auto result = make_uniq<TestAllTypesBindData>();
+	bool use_large_enum = false;
+	auto entry = input.named_parameters.find("use_large_enum");
+	if (entry != input.named_parameters.end()) {
+		use_large_enum = BooleanValue::Get(entry->second);
+	}
+	result->test_types = TestAllTypesFun::GetTestTypes(use_large_enum);
+	for (auto &test_type : result->test_types) {
+		return_types.push_back(test_type.type);
+		names.push_back(test_type.name);
 	}
-	return nullptr;
+	return std::move(result);
 }
 
 unique_ptr<GlobalTableFunctionState> TestAllTypesInit(ClientContext &context, TableFunctionInitInput &input) {
+	auto &bind_data = input.bind_data->Cast<TestAllTypesBindData>();
 	auto result = make_uniq<TestAllTypesData>();
-	auto test_types = TestAllTypesFun::GetTestTypes();
 	// 3 rows: min, max and NULL
 	result->entries.resize(3);
 	// initialize the values
-	for (auto &test_type : test_types) {
-		result->entries[0].push_back(std::move(test_type.min_value));
-		result->entries[1].push_back(std::move(test_type.max_value));
-		result->entries[2].emplace_back(std::move(test_type.type));
+	for (auto &test_type : bind_data.test_types) {
+		result->entries[0].push_back(test_type.min_value);
+		result->entries[1].push_back(test_type.max_value);
+		result->entries[2].emplace_back(test_type.type);
 	}
 	return std::move(result);
 }
@@ -243,7 +268,9 @@ void TestAllTypesFunction(ClientContext &context, TableFunctionInput &data_p, Da
 }
 
 void TestAllTypesFun::RegisterFunction(BuiltinFunctions &set) {
-	set.AddFunction(TableFunction("test_all_types", {}, TestAllTypesFunction, TestAllTypesBind, TestAllTypesInit));
+	TableFunction test_all_types("test_all_types", {}, TestAllTypesFunction, TestAllTypesBind, TestAllTypesInit);
+	test_all_types.named_parameters["use_large_enum"] = LogicalType::BOOLEAN;
+	set.AddFunction(test_all_types);
 }
 
 } // namespace duckdb

package/src/duckdb/src/function/table/system_functions.cpp

@@ -10,6 +10,7 @@ namespace duckdb {
 
 void BuiltinFunctions::RegisterSQLiteFunctions() {
 	PragmaVersion::RegisterFunction(*this);
+	PragmaPlatform::RegisterFunction(*this);
 	PragmaCollations::RegisterFunction(*this);
 	PragmaTableInfo::RegisterFunction(*this);
 	PragmaStorageInfo::RegisterFunction(*this);

package/src/duckdb/src/function/table/table_scan.cpp

@@ -15,6 +15,8 @@
 #include "duckdb/catalog/dependency_list.hpp"
 #include "duckdb/function/function_set.hpp"
 #include "duckdb/storage/table/scan_state.hpp"
+#include "duckdb/common/serializer/format_serializer.hpp"
+#include "duckdb/common/serializer/format_deserializer.hpp"
 
 namespace duckdb {
 
@@ -281,6 +283,15 @@ void TableScanPushdownComplexFilter(ClientContext &context, LogicalGet &get, Fun
 	if (bind_data.is_index_scan) {
 		return;
 	}
+	if (!get.table_filters.filters.empty()) {
+		// if there were filters before we can't convert this to an index scan
+		return;
+	}
+	if (!get.projection_ids.empty()) {
+		// if columns were pruned by RemoveUnusedColumns we can't convert this to an index scan,
+		// because index scan does not support filter_prune (yet)
+		return;
+	}
 	if (filters.empty()) {
 		// no indexes or no filters: skip the pushdown
 		return;
@@ -438,6 +449,35 @@ static unique_ptr<FunctionData> TableScanDeserialize(PlanDeserializationState &s
 	return std::move(result);
 }
 
+static void TableScanFormatSerialize(FormatSerializer &serializer, const optional_ptr<FunctionData> bind_data_p,
+                                     const TableFunction &function) {
+	auto &bind_data = bind_data_p->Cast<TableScanBindData>();
+	serializer.WriteProperty(100, "catalog", bind_data.table.schema.catalog.GetName());
+	serializer.WriteProperty(101, "schema", bind_data.table.schema.name);
+	serializer.WriteProperty(102, "table", bind_data.table.name);
+	serializer.WriteProperty(103, "is_index_scan", bind_data.is_index_scan);
+	serializer.WriteProperty(104, "is_create_index", bind_data.is_create_index);
+	serializer.WriteProperty(105, "result_ids", bind_data.result_ids);
+	serializer.WriteProperty(106, "result_ids", bind_data.result_ids);
+}
+
+static unique_ptr<FunctionData> TableScanFormatDeserialize(FormatDeserializer &deserializer, TableFunction &function) {
+	auto catalog = deserializer.ReadProperty<string>(100, "catalog");
+	auto schema = deserializer.ReadProperty<string>(101, "schema");
+	auto table = deserializer.ReadProperty<string>(102, "table");
+	auto &catalog_entry =
+	    Catalog::GetEntry<TableCatalogEntry>(deserializer.Get<ClientContext &>(), catalog, schema, table);
+	if (catalog_entry.type != CatalogType::TABLE_ENTRY) {
+		throw SerializationException("Cant find table for %s.%s", schema, table);
+	}
+	auto result = make_uniq<TableScanBindData>(catalog_entry.Cast<DuckTableEntry>());
+	deserializer.ReadProperty(103, "is_index_scan", result->is_index_scan);
+	deserializer.ReadProperty(104, "is_create_index", result->is_create_index);
+	deserializer.ReadProperty(105, "result_ids", result->result_ids);
+	deserializer.ReadProperty(106, "result_ids", result->result_ids);
+	return std::move(result);
+}
+
 TableFunction TableScanFunction::GetIndexScanFunction() {
 	TableFunction scan_function("index_scan", {}, IndexScanFunction);
 	scan_function.init_local = nullptr;
@@ -453,6 +493,8 @@ TableFunction TableScanFunction::GetIndexScanFunction() {
 	scan_function.filter_pushdown = false;
 	scan_function.serialize = TableScanSerialize;
 	scan_function.deserialize = TableScanDeserialize;
+	scan_function.format_serialize = TableScanFormatSerialize;
+	scan_function.format_deserialize = TableScanFormatDeserialize;
 	return scan_function;
 }
 
@@ -473,6 +515,8 @@ TableFunction TableScanFunction::GetFunction() {
 	scan_function.filter_prune = true;
 	scan_function.serialize = TableScanSerialize;
 	scan_function.deserialize = TableScanDeserialize;
+	scan_function.format_serialize = TableScanFormatSerialize;
+	scan_function.format_deserialize = TableScanFormatDeserialize;
 	return scan_function;
 }
 

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,11 +1,12 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.8.2-dev33"
+#define DUCKDB_VERSION "0.8.2-dev3334"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "5e5bd3d4f7"
+#define DUCKDB_SOURCE_ID "6937aee250"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"
+#include "duckdb/common/string_util.hpp"
 
 #include <cstdint>
 
@@ -63,6 +64,13 @@ const char *DuckDB::LibraryVersion() {
 }
 
 string DuckDB::Platform() {
+#if defined(DUCKDB_CUSTOM_PLATFORM)
+	return DUCKDB_QUOTE_DEFINE(DUCKDB_CUSTOM_PLATFORM);
+#endif
+#if defined(DUCKDB_WASM_VERSION)
+	// DuckDB-Wasm requires CUSTOM_PLATFORM to be defined
+	static_assert(0, "DUCKDB_WASM_VERSION should rely on CUSTOM_PLATFORM being provided");
+#endif
 	string os = "linux";
 #if INTPTR_MAX == INT64_MAX
 	string arch = "amd64";
@@ -87,6 +95,9 @@ string DuckDB::Platform() {
 		postfix = "_gcc4";
 	}
 #endif
+#if defined(__ANDROID__)
+	postfix += "_android"; // using + because it may also be gcc4
+#endif
 #ifdef __MINGW32__
 	postfix = "_mingw";
 #endif
@@ -97,4 +108,40 @@ string DuckDB::Platform() {
 	return os + "_" + arch + postfix;
 }
 
+struct PragmaPlatformData : public GlobalTableFunctionState {
+	PragmaPlatformData() : finished(false) {
+	}
+
+	bool finished;
+};
+
+static unique_ptr<FunctionData> PragmaPlatformBind(ClientContext &context, TableFunctionBindInput &input,
+                                                   vector<LogicalType> &return_types, vector<string> &names) {
+	names.emplace_back("platform");
+	return_types.emplace_back(LogicalType::VARCHAR);
+	return nullptr;
+}
+
+static unique_ptr<GlobalTableFunctionState> PragmaPlatformInit(ClientContext &context, TableFunctionInitInput &input) {
+	return make_uniq<PragmaPlatformData>();
+}
+
+static void PragmaPlatformFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) {
+	auto &data = data_p.global_state->Cast<PragmaPlatformData>();
+	if (data.finished) {
+		// finished returning values
+		return;
+	}
+	output.SetCardinality(1);
+	output.SetValue(0, 0, DuckDB::Platform());
+	data.finished = true;
+}
+
+void PragmaPlatform::RegisterFunction(BuiltinFunctions &set) {
+	TableFunction pragma_platform("pragma_platform", {}, PragmaPlatformFunction);
+	pragma_platform.bind = PragmaPlatformBind;
+	pragma_platform.init_global = PragmaPlatformInit;
+	set.AddFunction(pragma_platform);
+}
+
 } // namespace duckdb

package/src/duckdb/src/function/table_function.cpp

@@ -18,8 +18,8 @@ TableFunction::TableFunction(string name, vector<LogicalType> arguments, table_f
       init_global(init_global), init_local(init_local), function(function), in_out_function(nullptr),
       in_out_function_final(nullptr), statistics(nullptr), dependency(nullptr), cardinality(nullptr),
       pushdown_complex_filter(nullptr), to_string(nullptr), table_scan_progress(nullptr), get_batch_index(nullptr),
-      get_batch_info(nullptr), serialize(nullptr), deserialize(nullptr), projection_pushdown(false),
-      filter_pushdown(false), filter_prune(false) {
+      get_batch_info(nullptr), serialize(nullptr), deserialize(nullptr), format_serialize(nullptr),
+      format_deserialize(nullptr), projection_pushdown(false), filter_pushdown(false), filter_prune(false) {
 }
 
 TableFunction::TableFunction(const vector<LogicalType> &arguments, table_function_t function,
@@ -32,7 +32,8 @@ TableFunction::TableFunction()
       init_local(nullptr), function(nullptr), in_out_function(nullptr), statistics(nullptr), dependency(nullptr),
       cardinality(nullptr), pushdown_complex_filter(nullptr), to_string(nullptr), table_scan_progress(nullptr),
       get_batch_index(nullptr), get_batch_info(nullptr), serialize(nullptr), deserialize(nullptr),
-      projection_pushdown(false), filter_pushdown(false), filter_prune(false) {
+      format_serialize(nullptr), format_deserialize(nullptr), projection_pushdown(false), filter_pushdown(false),
+      filter_prune(false) {
 }
 
 bool TableFunction::Equal(const TableFunction &rhs) const {

package/src/duckdb/src/include/duckdb/catalog/catalog_entry/index_catalog_entry.hpp

@@ -10,7 +10,7 @@
 
 #include "duckdb/catalog/standard_entry.hpp"
 #include "duckdb/parser/parsed_data/create_index_info.hpp"
-#include "duckdb/storage/meta_block_writer.hpp"
+#include "duckdb/storage/metadata/metadata_writer.hpp"
 
 namespace duckdb {
 
@@ -31,11 +31,11 @@ public:
 	string sql;
 	vector<unique_ptr<ParsedExpression>> expressions;
 	vector<unique_ptr<ParsedExpression>> parsed_expressions;
+	case_insensitive_map_t<Value> options;
 
 public:
+	unique_ptr<CreateInfo> GetInfo() const override;
 	string ToSQL() const override;
-	void Serialize(Serializer &serializer) const;
-	static unique_ptr<CreateIndexInfo> Deserialize(Deserializer &source, ClientContext &context);
 
 	virtual string GetSchemaName() const = 0;
 	virtual string GetTableName() const = 0;