duckdb 0.8.2-dev4623.0 → 0.8.2-dev4653.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.8.2-dev4623.0",
+  "version": "0.8.2-dev4653.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/extension/json/buffered_json_reader.cpp

@@ -1,9 +1,8 @@
 #include "buffered_json_reader.hpp"
 
 #include "duckdb/common/file_opener.hpp"
-#include "duckdb/common/printer.hpp"
-#include "duckdb/common/serializer/serializer.hpp"
 #include "duckdb/common/serializer/deserializer.hpp"
+#include "duckdb/common/serializer/serializer.hpp"
 
 #include <utility>
 
@@ -24,7 +23,7 @@ bool JSONFileHandle::IsOpen() const {
 }
 
 void JSONFileHandle::Close() {
-	if (IsOpen()) {
+	if (IsOpen() && plain_file_source) {
 		file_handle->Close();
 		file_handle = nullptr;
 	}
@@ -174,12 +173,13 @@ BufferedJSONReader::BufferedJSONReader(ClientContext &context, BufferedJSONReade
 }
 
 void BufferedJSONReader::OpenJSONFile() {
-	D_ASSERT(!IsOpen());
 	lock_guard<mutex> guard(lock);
-	auto &file_system = FileSystem::GetFileSystem(context);
-	auto regular_file_handle =
-	    file_system.OpenFile(file_name.c_str(), FileFlags::FILE_FLAGS_READ, FileLockType::NO_LOCK, options.compression);
-	file_handle = make_uniq<JSONFileHandle>(std::move(regular_file_handle), BufferAllocator::Get(context));
+	if (!IsOpen()) {
+		auto &file_system = FileSystem::GetFileSystem(context);
+		auto regular_file_handle = file_system.OpenFile(file_name.c_str(), FileFlags::FILE_FLAGS_READ,
+		                                                FileLockType::NO_LOCK, options.compression);
+		file_handle = make_uniq<JSONFileHandle>(std::move(regular_file_handle), BufferAllocator::Get(context));
+	}
 	Reset();
 }
 

package/src/duckdb/extension/json/json_functions/read_json.cpp

@@ -17,6 +17,7 @@ void JSONScan::AutoDetect(ClientContext &context, JSONScanData &bind_data, vecto
 	Vector string_vector(LogicalType::VARCHAR);
 
 	// Loop through the files (if union_by_name, else just sample the first file)
+	idx_t remaining = bind_data.sample_size;
 	for (idx_t file_idx = 0; file_idx < bind_data.files.size(); file_idx++) {
 		// Create global/local state and place the reader in the right field
 		JSONScanGlobalState gstate(context, bind_data);
@@ -28,7 +29,6 @@ void JSONScan::AutoDetect(ClientContext &context, JSONScanData &bind_data, vecto
 		}
 
 		// Read and detect schema
-		idx_t remaining = bind_data.sample_size;
 		while (remaining != 0) {
 			allocator.Reset();
 			auto read_count = lstate.ReadNext(gstate);
@@ -56,7 +56,11 @@ void JSONScan::AutoDetect(ClientContext &context, JSONScanData &bind_data, vecto
 		}
 
 		// Close the file and stop detection if not union_by_name
-		if (!bind_data.options.file_options.union_by_name) {
+		if (bind_data.options.file_options.union_by_name) {
+			// When union_by_name=true we sample sample_size per file
+			remaining = bind_data.sample_size;
+		} else if (remaining == 0) {
+			// When union_by_name=false, we sample sample_size in total (across the first files)
 			break;
 		}
 	}

package/src/duckdb/extension/json/json_scan.cpp

@@ -2,11 +2,11 @@
 
 #include "duckdb/common/enum_util.hpp"
 #include "duckdb/common/multi_file_reader.hpp"
+#include "duckdb/common/serializer/deserializer.hpp"
+#include "duckdb/common/serializer/serializer.hpp"
 #include "duckdb/main/extension_helper.hpp"
 #include "duckdb/parallel/task_scheduler.hpp"
 #include "duckdb/storage/buffer_manager.hpp"
-#include "duckdb/common/serializer/serializer.hpp"
-#include "duckdb/common/serializer/deserializer.hpp"
 
 namespace duckdb {
 
@@ -558,10 +558,8 @@ bool JSONScanLocalState::ReadNextBuffer(JSONScanGlobalState &gstate) {
 		if (current_reader) {
 			// If we performed the final read of this reader in the previous iteration, close it now
 			if (is_last) {
-				if (gstate.bind_data.type != JSONScanType::SAMPLE) {
-					TryIncrementFileIndex(gstate);
-					current_reader->CloseJSONFile();
-				}
+				TryIncrementFileIndex(gstate);
+				current_reader->CloseJSONFile();
 				current_reader = nullptr;
 				continue;
 			}

package/src/duckdb/src/execution/operator/csv_scanner/csv_reader_options.cpp

@@ -500,7 +500,7 @@ void CSVReaderOptions::ToNamedParameters(named_parameter_map_t &named_params) {
 	}
 
 	named_params["normalize_names"] = Value::BOOLEAN(normalize_names);
-	if (!name_list.empty()) {
+	if (!name_list.empty() && !named_params.count("column_names") && !named_params.count("names")) {
 		named_params["column_names"] = StringVectorToValue(name_list);
 	}
 	named_params["all_varchar"] = Value::BOOLEAN(all_varchar);

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.8.2-dev4623"
+#define DUCKDB_VERSION "0.8.2-dev4653"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "52a47a6b31"
+#define DUCKDB_SOURCE_ID "bb287d4b22"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb.h

@@ -1740,7 +1740,7 @@ DUCKDB_API duckdb_vector duckdb_struct_vector_get_child(duckdb_vector vector, id
 /*!
 Returns whether or not a row is valid (i.e. not NULL) in the given validity mask.
 
-* validity: The validity mask, as obtained through `duckdb_data_chunk_get_validity`
+* validity: The validity mask, as obtained through `duckdb_vector_get_validity`
 * row: The row index
 * returns: true if the row is valid, false otherwise
 */
@@ -1749,10 +1749,10 @@ DUCKDB_API bool duckdb_validity_row_is_valid(uint64_t *validity, idx_t row);
 /*!
 In a validity mask, sets a specific row to either valid or invalid.
 
-Note that `duckdb_data_chunk_ensure_validity_writable` should be called before calling `duckdb_data_chunk_get_validity`,
+Note that `duckdb_vector_ensure_validity_writable` should be called before calling `duckdb_vector_get_validity`,
 to ensure that there is a validity mask to write to.
 
-* validity: The validity mask, as obtained through `duckdb_data_chunk_get_validity`.
+* validity: The validity mask, as obtained through `duckdb_vector_get_validity`.
 * row: The row index
 * valid: Whether or not to set the row to valid, or invalid
 */

package/src/duckdb/src/storage/checkpoint_manager.cpp

@@ -131,11 +131,11 @@ void SingleFileCheckpointWriter::CreateCheckpoint() {
 		throw FatalException("Checkpoint aborted before truncate because of PRAGMA checkpoint_abort flag");
 	}
 
-	// truncate the WAL
-	wal->Truncate(0);
-
 	// truncate the file
 	block_manager.Truncate();
+
+	// truncate the WAL
+	wal->Truncate(0);
 }
 
 void CheckpointReader::LoadCheckpoint(ClientContext &context, MetadataReader &reader) {

package/src/duckdb/src/storage/table/table_statistics.cpp

@@ -102,9 +102,7 @@ void TableStatistics::CopyStats(TableStatistics &other) {
 }
 
 void TableStatistics::Serialize(Serializer &serializer) const {
-	auto column_count = column_stats.size();
-	serializer.WriteList(100, "column_stats", column_count,
-	                     [&](Serializer::List &list, idx_t i) { list.WriteElement(column_stats[i]); });
+	serializer.WriteProperty(100, "column_stats", column_stats);
 }
 
 void TableStatistics::Deserialize(Deserializer &deserializer, ColumnList &columns) {

package/src/duckdb/src/storage/wal_replay.cpp

@@ -57,7 +57,10 @@ bool WriteAheadLog::Replay(AttachedDatabase &database, string &path) {
 				deserializer.End();
 			}
 		}
-	} catch (std::exception &ex) { // LCOV_EXCL_START
+	} catch (SerializationException &ex) { // LCOV_EXCL_START
+		                                   // serialization exception - torn WAL
+		                                   // continue reading
+	} catch (std::exception &ex) {
 		Printer::PrintF("Exception in WAL playback during initial read: %s\n", ex.what());
 		return false;
 	} catch (...) {
@@ -104,7 +107,10 @@ bool WriteAheadLog::Replay(AttachedDatabase &database, string &path) {
 				deserializer.End();
 			}
 		}
-	} catch (std::exception &ex) { // LCOV_EXCL_START
+	} catch (SerializationException &ex) { // LCOV_EXCL_START
+		// serialization error during WAL replay: rollback
+		con.Rollback();
+	} catch (std::exception &ex) {
 		// FIXME: this should report a proper warning in the connection
 		Printer::PrintF("Exception in WAL playback: %s\n", ex.what());
 		// exception thrown in WAL replay: rollback