duckdb 0.8.2-dev4711.0 → 0.8.2-dev5002.0

package/src/duckdb/src/storage/data_table.cpp

@@ -748,6 +748,9 @@ void DataTable::Append(DataChunk &chunk, TableAppendState &state) {
 }
 
 void DataTable::ScanTableSegment(idx_t row_start, idx_t count, const std::function<void(DataChunk &chunk)> &function) {
+	if (count == 0) {
+		return;
+	}
 	idx_t end = row_start + count;
 
 	vector<column_t> column_ids;
@@ -817,26 +820,24 @@ void DataTable::CommitAppend(transaction_t commit_id, idx_t row_start, idx_t cou
 	info->cardinality += count;
 }
 
-void DataTable::RevertAppendInternal(idx_t start_row, idx_t count) {
-	if (count == 0) {
-		// nothing to revert!
-		return;
-	}
+void DataTable::RevertAppendInternal(idx_t start_row) {
 	// adjust the cardinality
 	info->cardinality = start_row;
 	D_ASSERT(is_root);
 	// revert appends made to row_groups
-	row_groups->RevertAppendInternal(start_row, count);
+	row_groups->RevertAppendInternal(start_row);
 }
 
 void DataTable::RevertAppend(idx_t start_row, idx_t count) {
 	lock_guard<mutex> lock(append_lock);
 
+	// revert any appends to indexes
 	if (!info->indexes.Empty()) {
 		idx_t current_row_base = start_row;
 		row_t row_data[STANDARD_VECTOR_SIZE];
 		Vector row_identifiers(LogicalType::ROW_TYPE, data_ptr_cast(row_data));
-		ScanTableSegment(start_row, count, [&](DataChunk &chunk) {
+		idx_t scan_count = MinValue<idx_t>(count, row_groups->GetTotalRows() - start_row);
+		ScanTableSegment(start_row, scan_count, [&](DataChunk &chunk) {
 			for (idx_t i = 0; i < chunk.size(); i++) {
 				row_data[i] = current_row_base + i;
 			}
@@ -847,7 +848,16 @@ void DataTable::RevertAppend(idx_t start_row, idx_t count) {
 			current_row_base += chunk.size();
 		});
 	}
-	RevertAppendInternal(start_row, count);
+
+	// we need to vacuum the indexes to remove any buffers that are now empty
+	// due to reverting the appends
+	info->indexes.Scan([&](Index &index) {
+		index.Vacuum();
+		return false;
+	});
+
+	// revert the data table append
+	RevertAppendInternal(start_row);
 }
 
 //===--------------------------------------------------------------------===//

package/src/duckdb/src/storage/local_storage.cpp

@@ -178,7 +178,7 @@ void LocalTableStorage::AppendToIndexes(DuckTransaction &transaction, TableAppen
 			return true;
 		});
 		if (append_to_table) {
-			table.RevertAppendInternal(append_state.row_start, append_count);
+			table.RevertAppendInternal(append_state.row_start);
 		}
 
 		// we need to vacuum the indexes to remove any buffers that are now empty
@@ -353,7 +353,6 @@ void LocalStorage::Append(LocalAppendState &state, DataChunk &chunk) {
 
 	//! Append the chunk to the local storage
 	auto new_row_group = storage->row_groups->Append(chunk, state.append_state);
-
 	//! Check if we should pre-emptively flush blocks to disk
 	if (new_row_group) {
 		storage->WriteNewRowGroup();
@@ -432,6 +431,7 @@ void LocalStorage::Flush(DataTable &table, LocalTableStorage &storage) {
 
 	TableAppendState append_state;
 	table.AppendLock(append_state);
+	transaction.PushAppend(table, append_state.row_start, append_count);
 	if ((append_state.row_start == 0 || storage.row_groups->GetTotalRows() >= MERGE_THRESHOLD) &&
 	    storage.deleted_rows == 0) {
 		// table is currently empty OR we are bulk appending: move over the storage directly
@@ -453,7 +453,6 @@ void LocalStorage::Flush(DataTable &table, LocalTableStorage &storage) {
 		// append to the indexes and append to the base table
 		storage.AppendToIndexes(transaction, append_state, append_count, true);
 	}
-	transaction.PushAppend(table, append_state.row_start, append_count);
 
 	// possibly vacuum any excess index data
 	table.info->indexes.Scan([&](Index &index) {

package/src/duckdb/src/storage/serialization/serialize_logical_operator.cpp

@@ -314,16 +314,12 @@ unique_ptr<LogicalOperator> LogicalCreateIndex::Deserialize(Deserializer &deseri
 
 void LogicalCreateTable::Serialize(Serializer &serializer) const {
 	LogicalOperator::Serialize(serializer);
-	serializer.WriteProperty(200, "catalog", schema.ParentCatalog().GetName());
-	serializer.WriteProperty(201, "schema", schema.name);
-	serializer.WriteProperty(202, "info", info->base);
+	serializer.WriteProperty(200, "info", info->base);
 }
 
 unique_ptr<LogicalOperator> LogicalCreateTable::Deserialize(Deserializer &deserializer) {
-	auto catalog = deserializer.ReadProperty<string>(200, "catalog");
-	auto schema = deserializer.ReadProperty<string>(201, "schema");
-	auto info = deserializer.ReadProperty<unique_ptr<CreateInfo>>(202, "info");
-	auto result = duckdb::unique_ptr<LogicalCreateTable>(new LogicalCreateTable(deserializer.Get<ClientContext &>(), catalog, schema, std::move(info)));
+	auto info = deserializer.ReadProperty<unique_ptr<CreateInfo>>(200, "info");
+	auto result = duckdb::unique_ptr<LogicalCreateTable>(new LogicalCreateTable(deserializer.Get<ClientContext &>(), std::move(info)));
 	return std::move(result);
 }
 
@@ -338,22 +334,18 @@ unique_ptr<LogicalOperator> LogicalCrossProduct::Deserialize(Deserializer &deser
 
 void LogicalDelete::Serialize(Serializer &serializer) const {
 	LogicalOperator::Serialize(serializer);
-	serializer.WriteProperty(200, "catalog", table.ParentCatalog().GetName());
-	serializer.WriteProperty(201, "schema", table.ParentSchema().name);
-	serializer.WriteProperty(202, "table", table.name);
-	serializer.WriteProperty(203, "table_index", table_index);
-	serializer.WriteProperty(204, "return_chunk", return_chunk);
-	serializer.WriteProperty(205, "expressions", expressions);
+	serializer.WriteProperty(200, "table_info", table.GetInfo());
+	serializer.WriteProperty(201, "table_index", table_index);
+	serializer.WriteProperty(202, "return_chunk", return_chunk);
+	serializer.WriteProperty(203, "expressions", expressions);
 }
 
 unique_ptr<LogicalOperator> LogicalDelete::Deserialize(Deserializer &deserializer) {
-	auto catalog = deserializer.ReadProperty<string>(200, "catalog");
-	auto schema = deserializer.ReadProperty<string>(201, "schema");
-	auto table = deserializer.ReadProperty<string>(202, "table");
-	auto result = duckdb::unique_ptr<LogicalDelete>(new LogicalDelete(deserializer.Get<ClientContext &>(), catalog, schema, table));
-	deserializer.ReadProperty(203, "table_index", result->table_index);
-	deserializer.ReadProperty(204, "return_chunk", result->return_chunk);
-	deserializer.ReadProperty(205, "expressions", result->expressions);
+	auto table_info = deserializer.ReadProperty<unique_ptr<CreateInfo>>(200, "table_info");
+	auto result = duckdb::unique_ptr<LogicalDelete>(new LogicalDelete(deserializer.Get<ClientContext &>(), table_info));
+	deserializer.ReadProperty(201, "table_index", result->table_index);
+	deserializer.ReadProperty(202, "return_chunk", result->return_chunk);
+	deserializer.ReadProperty(203, "expressions", result->expressions);
 	return std::move(result);
 }
 
@@ -456,50 +448,46 @@ unique_ptr<LogicalOperator> LogicalFilter::Deserialize(Deserializer &deserialize
 
 void LogicalInsert::Serialize(Serializer &serializer) const {
 	LogicalOperator::Serialize(serializer);
-	serializer.WriteProperty(200, "catalog", table.ParentCatalog().GetName());
-	serializer.WriteProperty(201, "schema", table.ParentSchema().name);
-	serializer.WriteProperty(202, "table", table.name);
-	serializer.WriteProperty(203, "insert_values", insert_values);
-	serializer.WriteProperty(204, "column_index_map", column_index_map);
-	serializer.WriteProperty(205, "expected_types", expected_types);
-	serializer.WriteProperty(206, "table_index", table_index);
-	serializer.WriteProperty(207, "return_chunk", return_chunk);
-	serializer.WriteProperty(208, "bound_defaults", bound_defaults);
-	serializer.WriteProperty(209, "action_type", action_type);
-	serializer.WriteProperty(210, "expected_set_types", expected_set_types);
-	serializer.WriteProperty(211, "on_conflict_filter", on_conflict_filter);
-	serializer.WritePropertyWithDefault(212, "on_conflict_condition", on_conflict_condition, unique_ptr<Expression>());
-	serializer.WritePropertyWithDefault(213, "do_update_condition", do_update_condition, unique_ptr<Expression>());
-	serializer.WriteProperty(214, "set_columns", set_columns);
-	serializer.WriteProperty(215, "set_types", set_types);
-	serializer.WriteProperty(216, "excluded_table_index", excluded_table_index);
-	serializer.WriteProperty(217, "columns_to_fetch", columns_to_fetch);
-	serializer.WriteProperty(218, "source_columns", source_columns);
-	serializer.WriteProperty(219, "expressions", expressions);
+	serializer.WriteProperty(200, "table_info", table.GetInfo());
+	serializer.WriteProperty(201, "insert_values", insert_values);
+	serializer.WriteProperty(202, "column_index_map", column_index_map);
+	serializer.WriteProperty(203, "expected_types", expected_types);
+	serializer.WriteProperty(204, "table_index", table_index);
+	serializer.WriteProperty(205, "return_chunk", return_chunk);
+	serializer.WriteProperty(206, "bound_defaults", bound_defaults);
+	serializer.WriteProperty(207, "action_type", action_type);
+	serializer.WriteProperty(208, "expected_set_types", expected_set_types);
+	serializer.WriteProperty(209, "on_conflict_filter", on_conflict_filter);
+	serializer.WritePropertyWithDefault(210, "on_conflict_condition", on_conflict_condition, unique_ptr<Expression>());
+	serializer.WritePropertyWithDefault(211, "do_update_condition", do_update_condition, unique_ptr<Expression>());
+	serializer.WriteProperty(212, "set_columns", set_columns);
+	serializer.WriteProperty(213, "set_types", set_types);
+	serializer.WriteProperty(214, "excluded_table_index", excluded_table_index);
+	serializer.WriteProperty(215, "columns_to_fetch", columns_to_fetch);
+	serializer.WriteProperty(216, "source_columns", source_columns);
+	serializer.WriteProperty(217, "expressions", expressions);
 }
 
 unique_ptr<LogicalOperator> LogicalInsert::Deserialize(Deserializer &deserializer) {
-	auto catalog = deserializer.ReadProperty<string>(200, "catalog");
-	auto schema = deserializer.ReadProperty<string>(201, "schema");
-	auto table = deserializer.ReadProperty<string>(202, "table");
-	auto result = duckdb::unique_ptr<LogicalInsert>(new LogicalInsert(deserializer.Get<ClientContext &>(), catalog, schema, table));
-	deserializer.ReadProperty(203, "insert_values", result->insert_values);
-	deserializer.ReadProperty(204, "column_index_map", result->column_index_map);
-	deserializer.ReadProperty(205, "expected_types", result->expected_types);
-	deserializer.ReadProperty(206, "table_index", result->table_index);
-	deserializer.ReadProperty(207, "return_chunk", result->return_chunk);
-	deserializer.ReadProperty(208, "bound_defaults", result->bound_defaults);
-	deserializer.ReadProperty(209, "action_type", result->action_type);
-	deserializer.ReadProperty(210, "expected_set_types", result->expected_set_types);
-	deserializer.ReadProperty(211, "on_conflict_filter", result->on_conflict_filter);
-	deserializer.ReadPropertyWithDefault(212, "on_conflict_condition", result->on_conflict_condition, unique_ptr<Expression>());
-	deserializer.ReadPropertyWithDefault(213, "do_update_condition", result->do_update_condition, unique_ptr<Expression>());
-	deserializer.ReadProperty(214, "set_columns", result->set_columns);
-	deserializer.ReadProperty(215, "set_types", result->set_types);
-	deserializer.ReadProperty(216, "excluded_table_index", result->excluded_table_index);
-	deserializer.ReadProperty(217, "columns_to_fetch", result->columns_to_fetch);
-	deserializer.ReadProperty(218, "source_columns", result->source_columns);
-	deserializer.ReadProperty(219, "expressions", result->expressions);
+	auto table_info = deserializer.ReadProperty<unique_ptr<CreateInfo>>(200, "table_info");
+	auto result = duckdb::unique_ptr<LogicalInsert>(new LogicalInsert(deserializer.Get<ClientContext &>(), std::move(table_info)));
+	deserializer.ReadProperty(201, "insert_values", result->insert_values);
+	deserializer.ReadProperty(202, "column_index_map", result->column_index_map);
+	deserializer.ReadProperty(203, "expected_types", result->expected_types);
+	deserializer.ReadProperty(204, "table_index", result->table_index);
+	deserializer.ReadProperty(205, "return_chunk", result->return_chunk);
+	deserializer.ReadProperty(206, "bound_defaults", result->bound_defaults);
+	deserializer.ReadProperty(207, "action_type", result->action_type);
+	deserializer.ReadProperty(208, "expected_set_types", result->expected_set_types);
+	deserializer.ReadProperty(209, "on_conflict_filter", result->on_conflict_filter);
+	deserializer.ReadPropertyWithDefault(210, "on_conflict_condition", result->on_conflict_condition, unique_ptr<Expression>());
+	deserializer.ReadPropertyWithDefault(211, "do_update_condition", result->do_update_condition, unique_ptr<Expression>());
+	deserializer.ReadProperty(212, "set_columns", result->set_columns);
+	deserializer.ReadProperty(213, "set_types", result->set_types);
+	deserializer.ReadProperty(214, "excluded_table_index", result->excluded_table_index);
+	deserializer.ReadProperty(215, "columns_to_fetch", result->columns_to_fetch);
+	deserializer.ReadProperty(216, "source_columns", result->source_columns);
+	deserializer.ReadProperty(217, "expressions", result->expressions);
 	return std::move(result);
 }
 
@@ -723,28 +711,24 @@ unique_ptr<LogicalOperator> LogicalUnnest::Deserialize(Deserializer &deserialize
 
 void LogicalUpdate::Serialize(Serializer &serializer) const {
 	LogicalOperator::Serialize(serializer);
-	serializer.WriteProperty(200, "catalog", table.ParentCatalog().GetName());
-	serializer.WriteProperty(201, "schema", table.ParentSchema().name);
-	serializer.WriteProperty(202, "table", table.name);
-	serializer.WriteProperty(203, "table_index", table_index);
-	serializer.WriteProperty(204, "return_chunk", return_chunk);
-	serializer.WriteProperty(205, "expressions", expressions);
-	serializer.WriteProperty(206, "columns", columns);
-	serializer.WriteProperty(207, "bound_defaults", bound_defaults);
-	serializer.WriteProperty(208, "update_is_del_and_insert", update_is_del_and_insert);
+	serializer.WriteProperty(200, "table_info", table.GetInfo());
+	serializer.WriteProperty(201, "table_index", table_index);
+	serializer.WriteProperty(202, "return_chunk", return_chunk);
+	serializer.WriteProperty(203, "expressions", expressions);
+	serializer.WriteProperty(204, "columns", columns);
+	serializer.WriteProperty(205, "bound_defaults", bound_defaults);
+	serializer.WriteProperty(206, "update_is_del_and_insert", update_is_del_and_insert);
 }
 
 unique_ptr<LogicalOperator> LogicalUpdate::Deserialize(Deserializer &deserializer) {
-	auto catalog = deserializer.ReadProperty<string>(200, "catalog");
-	auto schema = deserializer.ReadProperty<string>(201, "schema");
-	auto table = deserializer.ReadProperty<string>(202, "table");
-	auto result = duckdb::unique_ptr<LogicalUpdate>(new LogicalUpdate(deserializer.Get<ClientContext &>(), catalog, schema, table));
-	deserializer.ReadProperty(203, "table_index", result->table_index);
-	deserializer.ReadProperty(204, "return_chunk", result->return_chunk);
-	deserializer.ReadProperty(205, "expressions", result->expressions);
-	deserializer.ReadProperty(206, "columns", result->columns);
-	deserializer.ReadProperty(207, "bound_defaults", result->bound_defaults);
-	deserializer.ReadProperty(208, "update_is_del_and_insert", result->update_is_del_and_insert);
+	auto table_info = deserializer.ReadProperty<unique_ptr<CreateInfo>>(200, "table_info");
+	auto result = duckdb::unique_ptr<LogicalUpdate>(new LogicalUpdate(deserializer.Get<ClientContext &>(), table_info));
+	deserializer.ReadProperty(201, "table_index", result->table_index);
+	deserializer.ReadProperty(202, "return_chunk", result->return_chunk);
+	deserializer.ReadProperty(203, "expressions", result->expressions);
+	deserializer.ReadProperty(204, "columns", result->columns);
+	deserializer.ReadProperty(205, "bound_defaults", result->bound_defaults);
+	deserializer.ReadProperty(206, "update_is_del_and_insert", result->update_is_del_and_insert);
 	return std::move(result);
 }
 

package/src/duckdb/src/storage/storage_manager.cpp

@@ -235,8 +235,12 @@ void SingleFileStorageManager::CreateCheckpoint(bool delete_wal, bool force_chec
 	auto &config = DBConfig::Get(db);
 	if (wal->GetWALSize() > 0 || config.options.force_checkpoint || force_checkpoint) {
 		// we only need to checkpoint if there is anything in the WAL
-		SingleFileCheckpointWriter checkpointer(db, *block_manager);
-		checkpointer.CreateCheckpoint();
+		try {
+			SingleFileCheckpointWriter checkpointer(db, *block_manager);
+			checkpointer.CreateCheckpoint();
+		} catch (std::exception &ex) {
+			throw FatalException("Failed to create checkpoint because of error: %s", ex.what());
+		}
 	}
 	if (delete_wal) {
 		wal->Delete();

package/src/duckdb/src/storage/table/row_group.cpp

@@ -782,6 +782,12 @@ RowGroupPointer RowGroup::Checkpoint(RowGroupWriter &writer, TableStatistics &gl
 	vector<CompressionType> compression_types;
 	compression_types.reserve(columns.size());
 	for (idx_t column_idx = 0; column_idx < GetColumnCount(); column_idx++) {
+		auto &column = GetColumn(column_idx);
+		if (column.count != this->count) {
+			throw InternalException("Corrupted in-memory column - column with index %llu has misaligned count (row "
+			                        "group has %llu rows, column has %llu)",
+			                        column_idx, this->count.load(), column.count);
+		}
 		compression_types.push_back(writer.GetColumnCompressionType(column_idx));
 	}
 	auto result = WriteToDisk(writer.GetPartialBlockManager(), compression_types);

package/src/duckdb/src/storage/table/row_group_collection.cpp

@@ -33,6 +33,7 @@ void RowGroupSegmentTree::Initialize(PersistentTableData &data) {
 
 unique_ptr<RowGroup> RowGroupSegmentTree::LoadSegment() {
 	if (current_row_group >= max_row_group) {
+		reader.reset();
 		finished_loading = true;
 		return nullptr;
 	}
@@ -426,9 +427,9 @@ void RowGroupCollection::CommitAppend(transaction_t commit_id, idx_t row_start,
 	}
 }
 
-void RowGroupCollection::RevertAppendInternal(idx_t start_row, idx_t count) {
-	if (total_rows != start_row + count) {
-		throw InternalException("Interleaved appends: this should no longer happen");
+void RowGroupCollection::RevertAppendInternal(idx_t start_row) {
+	if (total_rows <= start_row) {
+		return;
 	}
 	total_rows = start_row;
 

package/src/duckdb/src/storage/table/struct_column_data.cpp

@@ -128,6 +128,7 @@ void StructColumnData::Append(BaseStatistics &stats, ColumnAppendState &state, V
 		sub_columns[i]->Append(StructStats::GetChildStats(stats, i), state.child_appends[i + 1], *child_entries[i],
 		                       count);
 	}
+	this->count += count;
 }
 
 void StructColumnData::RevertAppend(row_t start_row) {
@@ -135,6 +136,7 @@ void StructColumnData::RevertAppend(row_t start_row) {
 	for (auto &sub_column : sub_columns) {
 		sub_column->RevertAppend(start_row);
 	}
+	this->count = start_row - this->start;
 }
 
 idx_t StructColumnData::Fetch(ColumnScanState &state, row_t row_id, Vector &result) {

package/src/duckdb/src/transaction/duck_transaction.cpp

@@ -142,6 +142,7 @@ string DuckTransaction::Commit(AttachedDatabase &db, transaction_t commit_id, bo
 		}
 		return string();
 	} catch (std::exception &ex) {
+		undo_buffer.RevertCommit(iterator_state, this->transaction_id);
 		return ex.what();
 	}
 }

package/src/duckdb/ub_src_common_row_operations.cpp

@@ -4,7 +4,7 @@
 
 #include "src/common/row_operations/row_gather.cpp"
 
-#include "src/common/row_operations/row_match.cpp"
+#include "src/common/row_operations/row_matcher.cpp"
 
 #include "src/common/row_operations/row_external.cpp"
 

package/src/statement.cpp

@@ -93,11 +93,9 @@ Statement::Statement(const Napi::CallbackInfo &info) : Napi::ObjectWrap<Statemen
 	int length = info.Length();
 
 	if (length <= 0 || !Connection::HasInstance(info[0])) {
-		Napi::TypeError::New(env, "Connection object expected").ThrowAsJavaScriptException();
-		return;
+		throw Napi::TypeError::New(env, "Connection object expected");
 	} else if (length <= 1 || !info[1].IsString()) {
-		Napi::TypeError::New(env, "SQL query expected").ThrowAsJavaScriptException();
-		return;
+		throw Napi::TypeError::New(env, "SQL query expected");
 	}
 
 	connection_ref = Napi::ObjectWrap<Connection>::Unwrap(info[0].As<Napi::Object>());

package/test/database_fail.test.ts

@@ -2,6 +2,7 @@ import * as sqlite3 from '..';
 import * as assert from 'assert';
 import {DuckDbError, RowData} from "..";
 import {Worker} from 'worker_threads';
+import {expect} from 'chai';
 
 describe('error handling', function() {
     var db: sqlite3.Database;
@@ -163,4 +164,9 @@ describe('error handling', function() {
       await run_worker(); // first should always succeed
       await run_worker(); // second fails without thread safety
     })
+
+    it("shouldn't crash on an exception", () => {
+        expect(() => new sqlite3.Database(':memory:', {file_search_path: '/'})).to.throw('Could not set option "file_search_path" as a global option');
+    });
 });
+