duckdb 0.9.1-dev0.0 → 0.9.1-dev19.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.9.1-dev0.0",
+  "version": "0.9.1-dev19.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/src/common/enum_util.cpp

@@ -68,7 +68,7 @@
 #include "duckdb/execution/index/art/node.hpp"
 #include "duckdb/execution/operator/scan/csv/base_csv_reader.hpp"
 #include "duckdb/execution/operator/scan/csv/csv_reader_options.hpp"
-#include "duckdb/execution/operator/scan/csv/csv_state_machine.hpp"
+#include "duckdb/execution/operator/scan/csv/csv_state.hpp"
 #include "duckdb/execution/operator/scan/csv/quote_rules.hpp"
 #include "duckdb/function/aggregate_state.hpp"
 #include "duckdb/function/function.hpp"

package/src/duckdb/src/common/serializer/binary_deserializer.cpp

@@ -8,7 +8,8 @@ namespace duckdb {
 void BinaryDeserializer::OnPropertyBegin(const field_id_t field_id, const char *) {
 	auto field = NextField();
 	if (field != field_id) {
-		throw InternalException("Failed to deserialize: field id mismatch, expected: %d, got: %d", field_id, field);
+		throw SerializationException("Failed to deserialize: field id mismatch, expected: %d, got: %d", field_id,
+		                             field);
 	}
 }
 
@@ -34,7 +35,8 @@ void BinaryDeserializer::OnObjectBegin() {
 void BinaryDeserializer::OnObjectEnd() {
 	auto next_field = NextField();
 	if (next_field != MESSAGE_TERMINATOR_FIELD_ID) {
-		throw InternalException("Failed to deserialize: expected end of object, but found field id: %d", next_field);
+		throw SerializationException("Failed to deserialize: expected end of object, but found field id: %d",
+		                             next_field);
 	}
 	nesting_level--;
 }

package/src/duckdb/src/execution/operator/csv_scanner/csv_state_machine_cache.cpp

@@ -3,8 +3,8 @@
 
 namespace duckdb {
 
-void InitializeTransitionArray(unsigned char *transition_array, const uint8_t state) {
-	for (uint32_t i = 0; i < NUM_TRANSITIONS; i++) {
+void InitializeTransitionArray(CSVState *transition_array, const CSVState state) {
+	for (uint32_t i = 0; i < StateMachine::NUM_TRANSITIONS; i++) {
 		transition_array[i] = state;
 	}
 }
@@ -13,72 +13,63 @@ void CSVStateMachineCache::Insert(const CSVStateMachineOptions &state_machine_op
 	D_ASSERT(state_machine_cache.find(state_machine_options) == state_machine_cache.end());
 	// Initialize transition array with default values to the Standard option
 	auto &transition_array = state_machine_cache[state_machine_options];
-	const uint8_t standard_state = static_cast<uint8_t>(CSVState::STANDARD);
-	const uint8_t field_separator_state = static_cast<uint8_t>(CSVState::DELIMITER);
-	const uint8_t record_separator_state = static_cast<uint8_t>(CSVState::RECORD_SEPARATOR);
-	const uint8_t carriage_return_state = static_cast<uint8_t>(CSVState::CARRIAGE_RETURN);
-	const uint8_t quoted_state = static_cast<uint8_t>(CSVState::QUOTED);
-	const uint8_t unquoted_state = static_cast<uint8_t>(CSVState::UNQUOTED);
-	const uint8_t escape_state = static_cast<uint8_t>(CSVState::ESCAPE);
-	const uint8_t empty_line_state = static_cast<uint8_t>(CSVState::EMPTY_LINE);
-	const uint8_t invalid_state = static_cast<uint8_t>(CSVState::INVALID);
 
-	for (uint32_t i = 0; i < NUM_STATES; i++) {
-		switch (i) {
-		case quoted_state:
-			InitializeTransitionArray(transition_array[i], quoted_state);
+	for (uint32_t i = 0; i < StateMachine::NUM_STATES; i++) {
+		CSVState cur_state = CSVState(i);
+		switch (cur_state) {
+		case CSVState::QUOTED:
+			InitializeTransitionArray(transition_array[cur_state], CSVState::QUOTED);
 			break;
-		case unquoted_state:
-		case invalid_state:
-		case escape_state:
-			InitializeTransitionArray(transition_array[i], invalid_state);
+		case CSVState::UNQUOTED:
+		case CSVState::INVALID:
+		case CSVState::ESCAPE:
+			InitializeTransitionArray(transition_array[cur_state], CSVState::INVALID);
 			break;
 		default:
-			InitializeTransitionArray(transition_array[i], standard_state);
+			InitializeTransitionArray(transition_array[cur_state], CSVState::STANDARD);
 			break;
 		}
 	}
 
 	// Now set values depending on configuration
 	// 1) Standard State
-	transition_array[standard_state][static_cast<uint8_t>(state_machine_options.delimiter)] = field_separator_state;
-	transition_array[standard_state][static_cast<uint8_t>('\n')] = record_separator_state;
-	transition_array[standard_state][static_cast<uint8_t>('\r')] = carriage_return_state;
-	transition_array[standard_state][static_cast<uint8_t>(state_machine_options.quote)] = quoted_state;
+	transition_array[CSVState::STANDARD][static_cast<uint8_t>(state_machine_options.delimiter)] = CSVState::DELIMITER;
+	transition_array[CSVState::STANDARD][static_cast<uint8_t>('\n')] = CSVState::RECORD_SEPARATOR;
+	transition_array[CSVState::STANDARD][static_cast<uint8_t>('\r')] = CSVState::CARRIAGE_RETURN;
+	transition_array[CSVState::STANDARD][static_cast<uint8_t>(state_machine_options.quote)] = CSVState::QUOTED;
 	// 2) Field Separator State
-	transition_array[field_separator_state][static_cast<uint8_t>(state_machine_options.delimiter)] =
-	    field_separator_state;
-	transition_array[field_separator_state][static_cast<uint8_t>('\n')] = record_separator_state;
-	transition_array[field_separator_state][static_cast<uint8_t>('\r')] = carriage_return_state;
-	transition_array[field_separator_state][static_cast<uint8_t>(state_machine_options.quote)] = quoted_state;
+	transition_array[CSVState::DELIMITER][static_cast<uint8_t>(state_machine_options.delimiter)] = CSVState::DELIMITER;
+	transition_array[CSVState::DELIMITER][static_cast<uint8_t>('\n')] = CSVState::RECORD_SEPARATOR;
+	transition_array[CSVState::DELIMITER][static_cast<uint8_t>('\r')] = CSVState::CARRIAGE_RETURN;
+	transition_array[CSVState::DELIMITER][static_cast<uint8_t>(state_machine_options.quote)] = CSVState::QUOTED;
 	// 3) Record Separator State
-	transition_array[record_separator_state][static_cast<uint8_t>(state_machine_options.delimiter)] =
-	    field_separator_state;
-	transition_array[record_separator_state][static_cast<uint8_t>('\n')] = empty_line_state;
-	transition_array[record_separator_state][static_cast<uint8_t>('\r')] = empty_line_state;
-	transition_array[record_separator_state][static_cast<uint8_t>(state_machine_options.quote)] = quoted_state;
+	transition_array[CSVState::RECORD_SEPARATOR][static_cast<uint8_t>(state_machine_options.delimiter)] =
+	    CSVState::DELIMITER;
+	transition_array[CSVState::RECORD_SEPARATOR][static_cast<uint8_t>('\n')] = CSVState::EMPTY_LINE;
+	transition_array[CSVState::RECORD_SEPARATOR][static_cast<uint8_t>('\r')] = CSVState::EMPTY_LINE;
+	transition_array[CSVState::RECORD_SEPARATOR][static_cast<uint8_t>(state_machine_options.quote)] = CSVState::QUOTED;
 	// 4) Carriage Return State
-	transition_array[carriage_return_state][static_cast<uint8_t>('\n')] = record_separator_state;
-	transition_array[carriage_return_state][static_cast<uint8_t>('\r')] = empty_line_state;
-	transition_array[carriage_return_state][static_cast<uint8_t>(state_machine_options.escape)] = escape_state;
+	transition_array[CSVState::CARRIAGE_RETURN][static_cast<uint8_t>('\n')] = CSVState::RECORD_SEPARATOR;
+	transition_array[CSVState::CARRIAGE_RETURN][static_cast<uint8_t>('\r')] = CSVState::EMPTY_LINE;
+	transition_array[CSVState::CARRIAGE_RETURN][static_cast<uint8_t>(state_machine_options.escape)] = CSVState::ESCAPE;
 	// 5) Quoted State
-	transition_array[quoted_state][static_cast<uint8_t>(state_machine_options.quote)] = unquoted_state;
+	transition_array[CSVState::QUOTED][static_cast<uint8_t>(state_machine_options.quote)] = CSVState::UNQUOTED;
 	if (state_machine_options.quote != state_machine_options.escape) {
-		transition_array[quoted_state][static_cast<uint8_t>(state_machine_options.escape)] = escape_state;
+		transition_array[CSVState::QUOTED][static_cast<uint8_t>(state_machine_options.escape)] = CSVState::ESCAPE;
 	}
 	// 6) Unquoted State
-	transition_array[unquoted_state][static_cast<uint8_t>('\n')] = record_separator_state;
-	transition_array[unquoted_state][static_cast<uint8_t>('\r')] = carriage_return_state;
-	transition_array[unquoted_state][static_cast<uint8_t>(state_machine_options.delimiter)] = field_separator_state;
+	transition_array[CSVState::UNQUOTED][static_cast<uint8_t>('\n')] = CSVState::RECORD_SEPARATOR;
+	transition_array[CSVState::UNQUOTED][static_cast<uint8_t>('\r')] = CSVState::CARRIAGE_RETURN;
+	transition_array[CSVState::UNQUOTED][static_cast<uint8_t>(state_machine_options.delimiter)] = CSVState::DELIMITER;
 	if (state_machine_options.quote == state_machine_options.escape) {
-		transition_array[unquoted_state][static_cast<uint8_t>(state_machine_options.escape)] = quoted_state;
+		transition_array[CSVState::UNQUOTED][static_cast<uint8_t>(state_machine_options.escape)] = CSVState::QUOTED;
 	}
 	// 7) Escaped State
-	transition_array[escape_state][static_cast<uint8_t>(state_machine_options.quote)] = quoted_state;
-	transition_array[escape_state][static_cast<uint8_t>(state_machine_options.escape)] = quoted_state;
+	transition_array[CSVState::ESCAPE][static_cast<uint8_t>(state_machine_options.quote)] = CSVState::QUOTED;
+	transition_array[CSVState::ESCAPE][static_cast<uint8_t>(state_machine_options.escape)] = CSVState::QUOTED;
 	// 8) Empty Line State
-	transition_array[empty_line_state][static_cast<uint8_t>('\r')] = empty_line_state;
-	transition_array[empty_line_state][static_cast<uint8_t>('\n')] = empty_line_state;
+	transition_array[CSVState::EMPTY_LINE][static_cast<uint8_t>('\r')] = CSVState::EMPTY_LINE;
+	transition_array[CSVState::EMPTY_LINE][static_cast<uint8_t>('\n')] = CSVState::EMPTY_LINE;
 }
 
 CSVStateMachineCache::CSVStateMachineCache() {
@@ -95,7 +86,7 @@ CSVStateMachineCache::CSVStateMachineCache() {
 	}
 }
 
-const state_machine_t &CSVStateMachineCache::Get(const CSVStateMachineOptions &state_machine_options) {
+const StateMachine &CSVStateMachineCache::Get(const CSVStateMachineOptions &state_machine_options) {
 	//! Custom State Machine, we need to create it and cache it first
 	if (state_machine_cache.find(state_machine_options) == state_machine_cache.end()) {
 		Insert(state_machine_options);

package/src/duckdb/src/execution/operator/csv_scanner/sniffer/dialect_detection.cpp

@@ -21,11 +21,7 @@ struct SniffDialect {
 			sniffed_column_counts.clear();
 			return true;
 		}
-		machine.pre_previous_state = machine.previous_state;
-		machine.previous_state = machine.state;
-
-		machine.state = static_cast<CSVState>(
-		    machine.transition_array[static_cast<uint8_t>(machine.state)][static_cast<uint8_t>(current_char)]);
+		machine.Transition(current_char);
 
 		bool carriage_return = machine.previous_state == CSVState::CARRIAGE_RETURN;
 		machine.column_count += machine.previous_state == CSVState::DELIMITER;

package/src/duckdb/src/execution/operator/csv_scanner/sniffer/header_detection.cpp

@@ -97,9 +97,14 @@ void CSVSniffer::DetectHeader() {
 	bool first_row_consistent = true;
 	// check if header row is all null and/or consistent with detected column data types
 	bool first_row_nulls = true;
-	// This case will fail in dialect detection, so we assert here just for sanity
-	D_ASSERT(best_candidate->options.null_padding ||
-	         best_sql_types_candidates_per_column_idx.size() == best_header_row.size());
+	// If null-padding is not allowed and there is a mismatch between our header candidate and the number of columns
+	// We can't detect the dialect/type options properly
+	if (!best_candidate->options.null_padding &&
+	    best_sql_types_candidates_per_column_idx.size() != best_header_row.size()) {
+		throw InvalidInputException(
+		    "Error in file \"%s\": CSV options could not be auto-detected. Consider setting parser options manually.",
+		    options.file_path);
+	}
 	for (idx_t col = 0; col < best_header_row.size(); col++) {
 		auto dummy_val = best_header_row[col];
 		if (!dummy_val.IsNull()) {

package/src/duckdb/src/execution/operator/csv_scanner/sniffer/type_detection.cpp

@@ -149,10 +149,8 @@ struct SniffValue {
 			sniffed_values[machine.cur_rows].set = true;
 			machine.line_start_pos = current_pos;
 		}
-		machine.pre_previous_state = machine.previous_state;
-		machine.previous_state = machine.state;
-		machine.state = static_cast<CSVState>(
-		    machine.transition_array[static_cast<uint8_t>(machine.state)][static_cast<uint8_t>(current_char)]);
+
+		machine.Transition(current_char);
 
 		bool carriage_return = machine.previous_state == CSVState::CARRIAGE_RETURN;
 		if (machine.previous_state == CSVState::DELIMITER ||

package/src/duckdb/src/execution/operator/csv_scanner/sniffer/type_refinement.cpp

@@ -14,10 +14,7 @@ struct Parse {
 
 	inline static bool Process(CSVStateMachine &machine, DataChunk &parse_chunk, char current_char, idx_t current_pos) {
 
-		machine.pre_previous_state = machine.previous_state;
-		machine.previous_state = machine.state;
-		machine.state = static_cast<CSVState>(
-		    machine.transition_array[static_cast<uint8_t>(machine.state)][static_cast<uint8_t>(current_char)]);
+		machine.Transition(current_char);
 
 		bool carriage_return = machine.previous_state == CSVState::CARRIAGE_RETURN;
 		if (machine.previous_state == CSVState::DELIMITER ||

package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp

@@ -261,7 +261,7 @@ idx_t RadixHTConfig::ExternalRadixBits(const idx_t &maximum_sink_radix_bits_p) {
 idx_t RadixHTConfig::SinkCapacity(ClientContext &context) {
 	// Get active and maximum number of threads
 	const idx_t active_threads = TaskScheduler::GetScheduler(context).NumberOfThreads();
-	const auto max_threads = DBConfig::GetSystemMaxThreads(FileSystem::GetFileSystem(context));
+	const auto max_threads = DBConfig::GetConfig(context).options.maximum_threads;
 
 	// Compute cache size per active thread (assuming cache is shared)
 	const auto total_shared_cache_size = max_threads * L3_CACHE_SIZE;

package/src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,8 +1,8 @@
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "0.9.0"
+#define DUCKDB_VERSION "0.9.1-dev19"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "0d84ccf478"
+#define DUCKDB_SOURCE_ID "1ea87567af"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

package/src/duckdb/src/include/duckdb/common/serializer/serialization_traits.hpp

@@ -1,6 +1,7 @@
 #pragma once
 #include <type_traits>
 #include <cstdint>
+#include <atomic>
 
 #include "duckdb/common/vector.hpp"
 #include "duckdb/common/unordered_map.hpp"

package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/csv_state.hpp

@@ -0,0 +1,28 @@
+//===----------------------------------------------------------------------===//
+//                         DuckDB
+//
+// duckdb/execution/operator/scan/csv/csv_state.hpp
+//
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include <cstdint>
+
+namespace duckdb {
+
+//! All States of CSV Parsing
+enum class CSVState : uint8_t {
+	STANDARD = 0,         //! Regular unquoted field state
+	DELIMITER = 1,        //! State after encountering a field separator (e.g., ;)
+	RECORD_SEPARATOR = 2, //! State after encountering a record separator (i.e., \n)
+	CARRIAGE_RETURN = 3,  //! State after encountering a carriage return(i.e., \r)
+	QUOTED = 4,           //! State when inside a quoted field
+	UNQUOTED = 5,         //! State when leaving a quoted field
+	ESCAPE = 6,           //! State when encountering an escape character (e.g., \)
+	EMPTY_LINE = 7,       //! State when encountering an empty line (i.e., \r\r \n\n, \n\r)
+	INVALID = 8           //! Got to an Invalid State, this should error.
+};
+
+} // namespace duckdb

package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/csv_state_machine.hpp

@@ -14,19 +14,6 @@
 
 namespace duckdb {
 
-//! All States of CSV Parsing
-enum class CSVState : uint8_t {
-	STANDARD = 0,         //! Regular unquoted field state
-	DELIMITER = 1,        //! State after encountering a field separator (e.g., ;)
-	RECORD_SEPARATOR = 2, //! State after encountering a record separator (i.e., \n)
-	CARRIAGE_RETURN = 3,  //! State after encountering a carriage return(i.e., \r)
-	QUOTED = 4,           //! State when inside a quoted field
-	UNQUOTED = 5,         //! State when leaving a quoted field
-	ESCAPE = 6,           //! State when encountering an escape character (e.g., \)
-	EMPTY_LINE = 7,       //! State when encountering an empty line (i.e., \r\r \n\n, \n\r)
-	INVALID = 8           //! Got to an Invalid State, this should error.
-};
-
 //! The CSV State Machine comprises a state transition array (STA).
 //! The STA indicates the current state of parsing based on both the current and preceding characters.
 //! This reveals whether we are dealing with a Field, a New Line, a Delimiter, and so forth.
@@ -38,6 +25,14 @@ public:
 	explicit CSVStateMachine(CSVReaderOptions &options_p, const CSVStateMachineOptions &state_machine_options,
 	                         shared_ptr<CSVBufferManager> buffer_manager_p,
 	                         CSVStateMachineCache &csv_state_machine_cache_p);
+
+	//! Transition all states to next state, that depends on the current char
+	inline void Transition(char current_char) {
+		pre_previous_state = previous_state;
+		previous_state = state;
+		state = transition_array[state][static_cast<uint8_t>(current_char)];
+	}
+
 	//! Resets the state machine, so it can be used again
 	void Reset();
 
@@ -52,7 +47,7 @@ public:
 	idx_t start_row = 0;
 	//! The Transition Array is a Finite State Machine
 	//! It holds the transitions of all states, on all 256 possible different characters
-	const state_machine_t &transition_array;
+	const StateMachine &transition_array;
 
 	//! Both these variables are used for new line identifier detection
 	bool single_record_separator = false;

package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/csv_state_machine_cache.hpp

@@ -8,14 +8,28 @@
 
 #pragma once
 
-#include "duckdb/execution/operator/scan/csv/csv_reader_options.hpp"
+#include "duckdb/execution/operator/scan/csv/csv_state.hpp"
 #include "duckdb/execution/operator/scan/csv/csv_buffer_manager.hpp"
+#include "duckdb/execution/operator/scan/csv/csv_reader_options.hpp"
 #include "duckdb/execution/operator/scan/csv/quote_rules.hpp"
 
 namespace duckdb {
-static constexpr uint32_t NUM_STATES = 9;
-static constexpr uint32_t NUM_TRANSITIONS = 256;
-typedef uint8_t state_machine_t[NUM_STATES][NUM_TRANSITIONS];
+
+//! Class to wrap the state machine matrix
+class StateMachine {
+public:
+	static constexpr uint32_t NUM_STATES = 9;
+	static constexpr uint32_t NUM_TRANSITIONS = 256;
+	CSVState state_machine[NUM_STATES][NUM_TRANSITIONS];
+
+	const CSVState *operator[](CSVState state) const {
+		return state_machine[static_cast<uint8_t>(state)];
+	}
+
+	CSVState *operator[](CSVState state) {
+		return state_machine[static_cast<uint8_t>(state)];
+	}
+};
 
 //! Hash function used in out state machine cache, it hashes and combines all options used to generate a state machine
 struct HashCSVStateMachineConfig {
@@ -36,12 +50,12 @@ public:
 	~CSVStateMachineCache() {};
 	//! Gets a state machine from the cache, if it's not from one the default options
 	//! It first caches it, then returns it.
-	const state_machine_t &Get(const CSVStateMachineOptions &state_machine_options);
+	const StateMachine &Get(const CSVStateMachineOptions &state_machine_options);
 
 private:
 	void Insert(const CSVStateMachineOptions &state_machine_options);
 	//! Cache on delimiter|quote|escape
-	unordered_map<CSVStateMachineOptions, state_machine_t, HashCSVStateMachineConfig> state_machine_cache;
+	unordered_map<CSVStateMachineOptions, StateMachine, HashCSVStateMachineConfig> state_machine_cache;
 	//! Default value for options used to intialize CSV State Machine Cache
 	const vector<char> default_delimiter = {',', '|', ';', '\t'};
 	const vector<vector<char>> default_quote = {{'\"'}, {'\"', '\''}, {'\0'}};