duckdb 0.9.3-dev2.0 → 0.10.0

package/src/duckdb/extension/json/json_functions/read_json.cpp

@@ -16,7 +16,7 @@ void JSONScan::AutoDetect(ClientContext &context, JSONScanData &bind_data, vecto
 	ArenaAllocator allocator(BufferAllocator::Get(context));
 	Vector string_vector(LogicalType::VARCHAR);
 
-	// Loop through the files (if union_by_name, else just sample the first file)
+	// Loop through the files (if union_by_name, else sample up to sample_size rows or maximum_sample_files files)
 	idx_t remaining = bind_data.sample_size;
 	for (idx_t file_idx = 0; file_idx < bind_data.files.size(); file_idx++) {
 		// Create global/local state and place the reader in the right field
@@ -46,7 +46,7 @@ void JSONScan::AutoDetect(ClientContext &context, JSONScanData &bind_data, vecto
 			if (!node.ContainsVarchar()) { // Can't refine non-VARCHAR types
 				continue;
 			}
-			node.InitializeCandidateTypes(bind_data.max_depth);
+			node.InitializeCandidateTypes(bind_data.max_depth, bind_data.convert_strings_to_integers);
 			node.RefineCandidateTypes(lstate.values, next, string_vector, allocator, bind_data.date_format_map);
 			remaining -= next;
 		}
@@ -59,8 +59,8 @@ void JSONScan::AutoDetect(ClientContext &context, JSONScanData &bind_data, vecto
 		if (bind_data.options.file_options.union_by_name) {
 			// When union_by_name=true we sample sample_size per file
 			remaining = bind_data.sample_size;
-		} else if (remaining == 0) {
-			// When union_by_name=false, we sample sample_size in total (across the first files)
+		} else if (remaining == 0 || file_idx == bind_data.maximum_sample_files - 1) {
+			// When union_by_name=false, we sample sample_size in total (across the first maximum_sample_files files)
 			break;
 		}
 	}
@@ -69,7 +69,8 @@ void JSONScan::AutoDetect(ClientContext &context, JSONScanData &bind_data, vecto
 	bind_data.type = JSONScanType::READ_JSON;
 
 	// Convert structure to logical type
-	auto type = JSONStructure::StructureToType(context, node, bind_data.max_depth);
+	auto type =
+	    JSONStructure::StructureToType(context, node, bind_data.max_depth, bind_data.field_appearance_threshold);
 
 	// Auto-detect record type
 	if (bind_data.options.record_type == JSONRecordType::AUTO_DETECT) {
@@ -115,6 +116,9 @@ unique_ptr<FunctionData> ReadJSONBind(ClientContext &context, TableFunctionBindI
 
 	for (auto &kv : input.named_parameters) {
 		auto loption = StringUtil::Lower(kv.first);
+		if (kv.second.IsNull()) {
+			throw BinderException("read_json parameter \"%s\" cannot be NULL.", loption);
+		}
 		if (loption == "columns") {
 			auto &child_type = kv.second.type();
 			if (child_type.id() != LogicalTypeId::STRUCT) {
@@ -145,8 +149,8 @@ unique_ptr<FunctionData> ReadJSONBind(ClientContext &context, TableFunctionBindI
 			} else if (arg > 0) {
 				bind_data->sample_size = arg;
 			} else {
-				throw BinderException(
-				    "read_json \"sample_size\" parameter must be positive, or -1 to sample the entire file.");
+				throw BinderException("read_json \"sample_size\" parameter must be positive, or -1 to sample all input "
+				                      "files entirely, up to \"maximum_sample_files\" files.");
 			}
 		} else if (loption == "maximum_depth") {
 			auto arg = BigIntValue::Get(kv.second);
@@ -155,6 +159,13 @@ unique_ptr<FunctionData> ReadJSONBind(ClientContext &context, TableFunctionBindI
 			} else {
 				bind_data->max_depth = arg;
 			}
+		} else if (loption == "field_appearance_threshold") {
+			auto arg = DoubleValue::Get(kv.second);
+			if (arg < 0 || arg > 1) {
+				throw BinderException(
+				    "read_json_auto \"field_appearance_threshold\" parameter must be between 0 and 1");
+			}
+			bind_data->field_appearance_threshold = arg;
 		} else if (loption == "dateformat" || loption == "date_format") {
 			auto format_string = StringValue::Get(kv.second);
 			if (StringUtil::Lower(format_string) == "iso") {
@@ -190,6 +201,18 @@ unique_ptr<FunctionData> ReadJSONBind(ClientContext &context, TableFunctionBindI
 			} else {
 				throw BinderException("read_json requires \"records\" to be one of ['auto', 'true', 'false'].");
 			}
+		} else if (loption == "maximum_sample_files") {
+			auto arg = BigIntValue::Get(kv.second);
+			if (arg == -1) {
+				bind_data->maximum_sample_files = NumericLimits<idx_t>::Maximum();
+			} else if (arg > 0) {
+				bind_data->maximum_sample_files = arg;
+			} else {
+				throw BinderException("read_json \"maximum_sample_files\" parameter must be positive, or -1 to remove "
+				                      "the limit on the number of files used to sample \"sample_size\" rows.");
+			}
+		} else if (loption == "convert_strings_to_integers") {
+			bind_data->convert_strings_to_integers = BooleanValue::Get(kv.second);
 		}
 	}
 
@@ -201,8 +224,8 @@ unique_ptr<FunctionData> ReadJSONBind(ClientContext &context, TableFunctionBindI
 	if (!bind_data->auto_detect) {
 		// Need to specify columns if RECORDS and not auto-detecting
 		if (return_types.empty()) {
-			throw BinderException("read_json requires columns to be specified through the \"columns\" parameter."
-			                      "\n Use read_json_auto or set auto_detect=true to automatically guess columns.");
+			throw BinderException("When auto_detect=false, read_json requires columns to be specified through the "
+			                      "\"columns\" parameter.");
 		}
 		// If we are reading VALUES, we can only have one column
 		if (bind_data->options.record_type == JSONRecordType::VALUES && return_types.size() != 1) {
@@ -305,6 +328,7 @@ TableFunction JSONFunctions::GetReadJSONTableFunction(shared_ptr<JSONScanInfo> f
 	table_function.named_parameters["timestampformat"] = LogicalType::VARCHAR;
 	table_function.named_parameters["timestamp_format"] = LogicalType::VARCHAR;
 	table_function.named_parameters["records"] = LogicalType::VARCHAR;
+	table_function.named_parameters["maximum_sample_files"] = LogicalType::BIGINT;
 
 	// TODO: might be able to do filter pushdown/prune ?
 
@@ -313,36 +337,37 @@ TableFunction JSONFunctions::GetReadJSONTableFunction(shared_ptr<JSONScanInfo> f
 	return table_function;
 }
 
-TableFunctionSet CreateJSONFunctionInfo(string name, shared_ptr<JSONScanInfo> info, bool auto_function = false) {
+TableFunctionSet CreateJSONFunctionInfo(string name, shared_ptr<JSONScanInfo> info) {
 	auto table_function = JSONFunctions::GetReadJSONTableFunction(std::move(info));
 	table_function.name = std::move(name);
-	if (auto_function) {
-		table_function.named_parameters["maximum_depth"] = LogicalType::BIGINT;
-	}
+	table_function.named_parameters["maximum_depth"] = LogicalType::BIGINT;
+	table_function.named_parameters["field_appearance_threshold"] = LogicalType::DOUBLE;
+	table_function.named_parameters["convert_strings_to_integers"] = LogicalType::BOOLEAN;
 	return MultiFileReader::CreateFunctionSet(table_function);
 }
 
 TableFunctionSet JSONFunctions::GetReadJSONFunction() {
-	auto info = make_shared<JSONScanInfo>(JSONScanType::READ_JSON, JSONFormat::ARRAY, JSONRecordType::RECORDS);
+	auto info =
+	    make_shared<JSONScanInfo>(JSONScanType::READ_JSON, JSONFormat::AUTO_DETECT, JSONRecordType::AUTO_DETECT, true);
 	return CreateJSONFunctionInfo("read_json", std::move(info));
 }
 
 TableFunctionSet JSONFunctions::GetReadNDJSONFunction() {
-	auto info =
-	    make_shared<JSONScanInfo>(JSONScanType::READ_JSON, JSONFormat::NEWLINE_DELIMITED, JSONRecordType::RECORDS);
+	auto info = make_shared<JSONScanInfo>(JSONScanType::READ_JSON, JSONFormat::NEWLINE_DELIMITED,
+	                                      JSONRecordType::AUTO_DETECT, true);
 	return CreateJSONFunctionInfo("read_ndjson", std::move(info));
 }
 
 TableFunctionSet JSONFunctions::GetReadJSONAutoFunction() {
 	auto info =
 	    make_shared<JSONScanInfo>(JSONScanType::READ_JSON, JSONFormat::AUTO_DETECT, JSONRecordType::AUTO_DETECT, true);
-	return CreateJSONFunctionInfo("read_json_auto", std::move(info), true);
+	return CreateJSONFunctionInfo("read_json_auto", std::move(info));
 }
 
 TableFunctionSet JSONFunctions::GetReadNDJSONAutoFunction() {
 	auto info = make_shared<JSONScanInfo>(JSONScanType::READ_JSON, JSONFormat::NEWLINE_DELIMITED,
 	                                      JSONRecordType::AUTO_DETECT, true);
-	return CreateJSONFunctionInfo("read_ndjson_auto", std::move(info), true);
+	return CreateJSONFunctionInfo("read_ndjson_auto", std::move(info));
 }
 
 } // namespace duckdb

package/src/duckdb/extension/json/json_functions/read_json_objects.cpp

@@ -10,7 +10,7 @@ unique_ptr<FunctionData> ReadJSONObjectsBind(ClientContext &context, TableFuncti
 	bind_data->Bind(context, input);
 
 	bind_data->names.emplace_back("json");
-	return_types.push_back(JSONCommon::JSONType());
+	return_types.push_back(LogicalType::JSON());
 	names.emplace_back("json");
 
 	bind_data->reader_bind =

package/src/duckdb/extension/json/json_functions.cpp

@@ -54,7 +54,7 @@ unique_ptr<FunctionData> JSONReadFunctionData::Bind(ClientContext &context, Scal
                                                     vector<unique_ptr<Expression>> &arguments) {
 	D_ASSERT(bound_function.arguments.size() == 2);
 	bool constant = false;
-	string path = "";
+	string path;
 	size_t len = 0;
 	JSONPathType path_type = JSONPathType::REGULAR;
 	if (arguments[1]->IsFoldable()) {
@@ -62,6 +62,7 @@ unique_ptr<FunctionData> JSONReadFunctionData::Bind(ClientContext &context, Scal
 		const auto path_val = ExpressionExecutor::EvaluateScalar(context, *arguments[1]);
 		path_type = CheckPath(path_val, path, len);
 	}
+	bound_function.arguments[1] = LogicalType::VARCHAR;
 	if (path_type == JSONCommon::JSONPathType::WILDCARD) {
 		bound_function.return_type = LogicalType::LIST(bound_function.return_type);
 	}
@@ -157,6 +158,7 @@ vector<ScalarFunctionSet> JSONFunctions::GetScalarFunctions() {
 	functions.push_back(GetKeysFunction());
 	functions.push_back(GetTypeFunction());
 	functions.push_back(GetValidFunction());
+	functions.push_back(GetSerializePlanFunction());
 	functions.push_back(GetSerializeSqlFunction());
 	functions.push_back(GetDeserializeSqlFunction());
 
@@ -227,6 +229,7 @@ static bool CastVarcharToJSON(Vector &source, Vector &result, idx_t count, CastP
 				    success = false;
 			    }
 		    }
+
 		    return input;
 	    });
 	StringVector::AddHeapReference(result, source);
@@ -235,16 +238,17 @@ static bool CastVarcharToJSON(Vector &source, Vector &result, idx_t count, CastP
 
 void JSONFunctions::RegisterSimpleCastFunctions(CastFunctionSet &casts) {
 	// JSON to VARCHAR is basically free
-	casts.RegisterCastFunction(JSONCommon::JSONType(), LogicalType::VARCHAR, DefaultCasts::ReinterpretCast, 1);
+	casts.RegisterCastFunction(LogicalType::JSON(), LogicalType::VARCHAR, DefaultCasts::ReinterpretCast, 1);
 
 	// VARCHAR to JSON requires a parse so it's not free. Let's make it 1 more than a cast to STRUCT
 	auto varchar_to_json_cost = casts.ImplicitCastCost(LogicalType::SQLNULL, LogicalTypeId::STRUCT) + 1;
-	BoundCastInfo info(CastVarcharToJSON, nullptr, JSONFunctionLocalState::InitCastLocalState);
-	casts.RegisterCastFunction(LogicalType::VARCHAR, JSONCommon::JSONType(), std::move(info), varchar_to_json_cost);
+	BoundCastInfo varchar_to_json_info(CastVarcharToJSON, nullptr, JSONFunctionLocalState::InitCastLocalState);
+	casts.RegisterCastFunction(LogicalType::VARCHAR, LogicalType::JSON(), std::move(varchar_to_json_info),
+	                           varchar_to_json_cost);
 
 	// Register NULL to JSON with a different cost than NULL to VARCHAR so the binder can disambiguate functions
 	auto null_to_json_cost = casts.ImplicitCastCost(LogicalType::SQLNULL, LogicalTypeId::VARCHAR) + 1;
-	casts.RegisterCastFunction(LogicalType::SQLNULL, JSONCommon::JSONType(), DefaultCasts::ReinterpretCast,
+	casts.RegisterCastFunction(LogicalType::SQLNULL, LogicalType::JSON(), DefaultCasts::TryVectorNullCast,
 	                           null_to_json_cost);
 }
 

package/src/duckdb/extension/json/json_scan.cpp

@@ -146,8 +146,8 @@ JSONScanGlobalState::JSONScanGlobalState(ClientContext &context, const JSONScanD
 
 JSONScanLocalState::JSONScanLocalState(ClientContext &context, JSONScanGlobalState &gstate)
     : scan_count(0), batch_index(DConstants::INVALID_INDEX), total_read_size(0), total_tuple_count(0),
-      bind_data(gstate.bind_data), allocator(BufferAllocator::Get(context)), current_reader(nullptr),
-      current_buffer_handle(nullptr), is_last(false), buffer_size(0), buffer_offset(0), prev_buffer_remainder(0) {
+      bind_data(gstate.bind_data), allocator(BufferAllocator::Get(context)), is_last(false),
+      fs(FileSystem::GetFileSystem(context)), buffer_size(0), buffer_offset(0), prev_buffer_remainder(0) {
 
 	// Buffer to reconstruct JSON values when they cross a buffer boundary
 	reconstruct_buffer = gstate.allocator.Allocate(gstate.buffer_capacity);
@@ -274,11 +274,11 @@ idx_t JSONScanLocalState::ReadNext(JSONScanGlobalState &gstate) {
 			if (!ReadNextBuffer(gstate)) {
 				break;
 			}
-			D_ASSERT(buffer_size != 0);
 			if (current_buffer_handle->buffer_index != 0 &&
 			    current_reader->GetFormat() == JSONFormat::NEWLINE_DELIMITED) {
-				ReconstructFirstObject();
-				scan_count++;
+				if (ReconstructFirstObject()) {
+					scan_count++;
+				}
 			}
 		}
 
@@ -288,14 +288,14 @@ idx_t JSONScanLocalState::ReadNext(JSONScanGlobalState &gstate) {
 	return scan_count;
 }
 
-static inline const char *NextNewline(char *ptr, idx_t size) {
-	return char_ptr_cast(memchr(ptr, '\n', size));
+static inline const char *NextNewline(const char *ptr, const idx_t size) {
+	return const_char_ptr_cast(memchr(ptr, '\n', size));
 }
 
-static inline const char *PreviousNewline(const char *ptr) {
-	for (ptr--; true; ptr--) {
-		const auto &c = *ptr;
-		if (c == '\n') {
+static inline const char *PreviousNewline(const char *ptr, const idx_t size) {
+	const auto end = ptr - size;
+	for (ptr--; ptr != end; ptr--) {
+		if (*ptr == '\n') {
 			break;
 		}
 	}
@@ -431,14 +431,9 @@ void JSONScanLocalState::ThrowObjectSizeError(const idx_t object_size) {
 	    bind_data.maximum_object_size, current_reader->GetFileName(), object_size);
 }
 
-void JSONScanLocalState::ThrowInvalidAtEndError() {
-	throw InvalidInputException("Invalid JSON detected at the end of file \"%s\".", current_reader->GetFileName());
-}
-
 void JSONScanLocalState::TryIncrementFileIndex(JSONScanGlobalState &gstate) const {
-	lock_guard<mutex> guard(gstate.lock);
 	if (gstate.file_index < gstate.json_readers.size() &&
-	    current_reader.get() == gstate.json_readers[gstate.file_index].get()) {
+	    RefersToSameObject(*current_reader, *gstate.json_readers[gstate.file_index])) {
 		gstate.file_index++;
 	}
 }
@@ -538,10 +533,10 @@ bool JSONScanLocalState::ReadNextBuffer(JSONScanGlobalState &gstate) {
 	AllocatedData buffer;
 
 	// Try to re-use a buffer that was used before
-	if (current_reader) {
-		current_reader->SetBufferLineOrObjectCount(current_buffer_handle->buffer_index, lines_or_objects_in_buffer);
-		if (current_buffer_handle && --current_buffer_handle->readers == 0) {
-			buffer = current_reader->RemoveBuffer(current_buffer_handle->buffer_index);
+	if (current_reader && current_buffer_handle) {
+		current_reader->SetBufferLineOrObjectCount(*current_buffer_handle, lines_or_objects_in_buffer);
+		if (--current_buffer_handle->readers == 0) {
+			buffer = current_reader->RemoveBuffer(*current_buffer_handle);
 		}
 	}
 
@@ -559,69 +554,75 @@ bool JSONScanLocalState::ReadNextBuffer(JSONScanGlobalState &gstate) {
 
 	optional_idx buffer_index;
 	while (true) {
-		// Now we finish the current reader
+		// Continue with the current reader
 		if (current_reader) {
-			// If we performed the final read of this reader in the previous iteration, close it now
-			if (is_last) {
-				TryIncrementFileIndex(gstate);
-				current_reader->CloseJSONFile();
-				current_reader = nullptr;
-				continue;
+			// Try to read (if we were not the last read in the previous iteration)
+			bool file_done = false;
+			bool read_success = ReadNextBufferInternal(gstate, buffer_index, file_done);
+			if (!is_last && read_success) {
+				// We read something
+				if (buffer_index.GetIndex() == 0 && current_reader->GetFormat() == JSONFormat::ARRAY) {
+					SkipOverArrayStart();
+				}
 			}
 
-			// Try to read
-			ReadNextBufferInternal(gstate, buffer_index);
-
-			// If this is the last read, end the parallel scan now so threads can move on
-			if (is_last && IsParallel(gstate)) {
+			if (file_done) {
+				lock_guard<mutex> guard(gstate.lock);
 				TryIncrementFileIndex(gstate);
+				current_reader->GetFileHandle().Close();
 			}
 
-			if (buffer_size == 0) {
-				// We didn't read anything, re-enter the loop
-				continue;
-			} else {
-				// We read something!
+			if (read_success) {
 				break;
 			}
+
+			// We were the last reader last time, or we didn't read anything this time
+			current_reader = nullptr;
+			current_buffer_handle = nullptr;
+			is_last = false;
 		}
+		D_ASSERT(!current_buffer_handle);
 
 		// If we got here, we don't have a reader (anymore). Try to get one
-		is_last = false;
-		{
-			lock_guard<mutex> guard(gstate.lock);
-			if (gstate.file_index == gstate.json_readers.size()) {
-				return false; // No more files left
-			}
+		unique_lock<mutex> guard(gstate.lock);
+		if (gstate.file_index == gstate.json_readers.size()) {
+			return false; // No more files left
+		}
 
-			// Assign the next reader to this thread
-			current_reader = gstate.json_readers[gstate.file_index].get();
+		// Assign the next reader to this thread
+		current_reader = gstate.json_readers[gstate.file_index].get();
 
-			// Open the file if it is not yet open
-			if (!current_reader->IsOpen()) {
-				current_reader->OpenJSONFile();
-			}
-			batch_index = gstate.batch_index++;
+		batch_index = gstate.batch_index++;
+		if (!gstate.enable_parallel_scans) {
+			// Non-parallel scans, increment file index and unlock
+			gstate.file_index++;
+			guard.unlock();
+		}
 
-			// Auto-detect format / record type
-			if (gstate.enable_parallel_scans) {
-				// Auto-detect within the lock, so threads may join a parallel NDJSON scan
-				if (current_reader->GetFormat() == JSONFormat::AUTO_DETECT) {
-					ReadAndAutoDetect(gstate, buffer_index);
+		// Open the file if it is not yet open
+		if (!current_reader->IsOpen()) {
+			current_reader->OpenJSONFile();
+			if (current_reader->GetFileHandle().FileSize() == 0) {
+				current_reader->GetFileHandle().Close();
+				// Skip over empty files
+				if (gstate.enable_parallel_scans) {
+					TryIncrementFileIndex(gstate);
 				}
-			} else {
-				gstate.file_index++; // Increment the file index before dropping lock so other threads move on
+				continue;
 			}
 		}
 
-		// If we didn't auto-detect within the lock, do it now
-		if (current_reader->GetFormat() == JSONFormat::AUTO_DETECT) {
+		// Auto-detect if we haven't yet done this during the bind
+		if (gstate.bind_data.options.record_type == JSONRecordType::AUTO_DETECT ||
+		    current_reader->GetFormat() == JSONFormat::AUTO_DETECT) {
 			ReadAndAutoDetect(gstate, buffer_index);
 		}
 
-		// If we haven't already, increment the file index if non-parallel scan
-		if (gstate.enable_parallel_scans && !IsParallel(gstate)) {
-			TryIncrementFileIndex(gstate);
+		if (gstate.enable_parallel_scans) {
+			if (!IsParallel(gstate)) {
+				// We still have the lock here if parallel scans are enabled
+				TryIncrementFileIndex(gstate);
+			}
 		}
 
 		if (!buffer_index.IsValid() || buffer_size == 0) {
@@ -631,7 +632,6 @@ bool JSONScanLocalState::ReadNextBuffer(JSONScanGlobalState &gstate) {
 
 		break;
 	}
-	D_ASSERT(buffer_size != 0); // We should have read something if we got here
 	D_ASSERT(buffer_index.IsValid());
 
 	idx_t readers = 1;
@@ -656,13 +656,18 @@ bool JSONScanLocalState::ReadNextBuffer(JSONScanGlobalState &gstate) {
 
 void JSONScanLocalState::ReadAndAutoDetect(JSONScanGlobalState &gstate, optional_idx &buffer_index) {
 	// We have to detect the JSON format - hold the gstate lock while we do this
-	ReadNextBufferInternal(gstate, buffer_index);
+	bool file_done = false;
+	if (!ReadNextBufferInternal(gstate, buffer_index, file_done)) {
+		return;
+	}
 	if (buffer_size == 0) {
 		return;
 	}
 
 	auto format_and_record_type = DetectFormatAndRecordType(buffer_ptr, buffer_size, allocator.GetYYAlc());
-	current_reader->SetFormat(format_and_record_type.first);
+	if (current_reader->GetFormat() == JSONFormat::AUTO_DETECT) {
+		current_reader->SetFormat(format_and_record_type.first);
+	}
 	if (current_reader->GetRecordType() == JSONRecordType::AUTO_DETECT) {
 		current_reader->SetRecordType(format_and_record_type.second);
 	}
@@ -677,20 +682,24 @@ void JSONScanLocalState::ReadAndAutoDetect(JSONScanGlobalState &gstate, optional
 	}
 }
 
-void JSONScanLocalState::ReadNextBufferInternal(JSONScanGlobalState &gstate, optional_idx &buffer_index) {
+bool JSONScanLocalState::ReadNextBufferInternal(JSONScanGlobalState &gstate, optional_idx &buffer_index,
+                                                bool &file_done) {
 	if (current_reader->GetFileHandle().CanSeek()) {
-		ReadNextBufferSeek(gstate, buffer_index);
+		if (!ReadNextBufferSeek(gstate, buffer_index, file_done)) {
+			return false;
+		}
 	} else {
-		ReadNextBufferNoSeek(gstate, buffer_index);
+		if (!ReadNextBufferNoSeek(gstate, buffer_index, file_done)) {
+			return false;
+		}
 	}
 
 	buffer_offset = 0;
-	if (buffer_index.GetIndex() == 0 && current_reader->GetFormat() == JSONFormat::ARRAY) {
-		SkipOverArrayStart();
-	}
+
+	return true;
 }
 
-void JSONScanLocalState::ReadNextBufferSeek(JSONScanGlobalState &gstate, optional_idx &buffer_index) {
+bool JSONScanLocalState::ReadNextBufferSeek(JSONScanGlobalState &gstate, optional_idx &buffer_index, bool &file_done) {
 	auto &file_handle = current_reader->GetFileHandle();
 
 	idx_t request_size = gstate.buffer_capacity - prev_buffer_remainder - YYJSON_PADDING_SIZE;
@@ -699,59 +708,61 @@ void JSONScanLocalState::ReadNextBufferSeek(JSONScanGlobalState &gstate, optiona
 
 	{
 		lock_guard<mutex> reader_guard(current_reader->lock);
-		buffer_index = current_reader->GetBufferIndex();
-
-		read_size = file_handle.GetPositionAndSize(read_position, request_size);
-		is_last = read_size < request_size;
-
-		if (!gstate.bind_data.ignore_errors && read_size == 0 && prev_buffer_remainder != 0) {
-			ThrowInvalidAtEndError();
+		if (!file_handle.GetPositionAndSize(read_position, read_size, request_size)) {
+			return false; // We weren't able to read
 		}
+		buffer_index = current_reader->GetBufferIndex();
+		is_last = read_size == 0;
 
-		if (read_size != 0 && current_reader->GetFormat() == JSONFormat::NEWLINE_DELIMITED) {
+		if (current_reader->GetFormat() == JSONFormat::NEWLINE_DELIMITED) {
 			batch_index = gstate.batch_index++;
 		}
 	}
 	buffer_size = prev_buffer_remainder + read_size;
-	if (buffer_size == 0) {
-		current_reader->SetBufferLineOrObjectCount(buffer_index.GetIndex(), 0);
-		return;
+
+	if (read_size != 0) {
+		auto &raw_handle = file_handle.GetHandle();
+		// For non-on-disk files, we create a handle per thread: this is faster for e.g. S3Filesystem where throttling
+		// per tcp connection can occur meaning that using multiple connections is faster.
+		if (!raw_handle.OnDiskFile() && raw_handle.CanSeek()) {
+			if (!thread_local_filehandle || thread_local_filehandle->GetPath() != raw_handle.GetPath()) {
+				thread_local_filehandle =
+				    fs.OpenFile(raw_handle.GetPath(), FileFlags::FILE_FLAGS_READ | FileFlags::FILE_FLAGS_DIRECT_IO);
+			}
+		} else if (thread_local_filehandle) {
+			thread_local_filehandle = nullptr;
+		}
 	}
 
 	// Now read the file lock-free!
-	file_handle.ReadAtPosition(buffer_ptr + prev_buffer_remainder, read_size, read_position,
-	                           gstate.bind_data.type == JSONScanType::SAMPLE);
+	file_handle.ReadAtPosition(buffer_ptr + prev_buffer_remainder, read_size, read_position, file_done,
+	                           gstate.bind_data.type == JSONScanType::SAMPLE, thread_local_filehandle);
+
+	return true;
 }
 
-void JSONScanLocalState::ReadNextBufferNoSeek(JSONScanGlobalState &gstate, optional_idx &buffer_index) {
+bool JSONScanLocalState::ReadNextBufferNoSeek(JSONScanGlobalState &gstate, optional_idx &buffer_index,
+                                              bool &file_done) {
 	idx_t request_size = gstate.buffer_capacity - prev_buffer_remainder - YYJSON_PADDING_SIZE;
 	idx_t read_size;
+
 	{
 		lock_guard<mutex> reader_guard(current_reader->lock);
-		buffer_index = current_reader->GetBufferIndex();
-
-		if (current_reader->HasFileHandle() && current_reader->IsOpen()) {
-			read_size = current_reader->GetFileHandle().Read(buffer_ptr + prev_buffer_remainder, request_size,
-			                                                 gstate.bind_data.type == JSONScanType::SAMPLE);
-			is_last = read_size < request_size;
-		} else {
-			read_size = 0;
-			is_last = true;
-		}
-
-		if (!gstate.bind_data.ignore_errors && read_size == 0 && prev_buffer_remainder != 0) {
-			ThrowInvalidAtEndError();
+		if (!current_reader->HasFileHandle() || !current_reader->IsOpen() ||
+		    !current_reader->GetFileHandle().Read(buffer_ptr + prev_buffer_remainder, read_size, request_size,
+		                                          file_done, gstate.bind_data.type == JSONScanType::SAMPLE)) {
+			return false; // Couldn't read anything
 		}
+		buffer_index = current_reader->GetBufferIndex();
+		is_last = read_size == 0;
 
-		if (read_size != 0 && current_reader->GetFormat() == JSONFormat::NEWLINE_DELIMITED) {
+		if (current_reader->GetFormat() == JSONFormat::NEWLINE_DELIMITED) {
 			batch_index = gstate.batch_index++;
 		}
 	}
 	buffer_size = prev_buffer_remainder + read_size;
-	if (buffer_size == 0) {
-		current_reader->SetBufferLineOrObjectCount(buffer_index.GetIndex(), 0);
-		return;
-	}
+
+	return true;
 }
 
 void JSONScanLocalState::SkipOverArrayStart() {
@@ -783,7 +794,7 @@ void JSONScanLocalState::SkipOverArrayStart() {
 	}
 }
 
-void JSONScanLocalState::ReconstructFirstObject() {
+bool JSONScanLocalState::ReconstructFirstObject() {
 	D_ASSERT(current_buffer_handle->buffer_index != 0);
 	D_ASSERT(current_reader->GetFormat() == JSONFormat::NEWLINE_DELIMITED);
 
@@ -795,37 +806,48 @@ void JSONScanLocalState::ReconstructFirstObject() {
 
 	// First we find the newline in the previous block
 	auto prev_buffer_ptr = char_ptr_cast(previous_buffer_handle->buffer.get()) + previous_buffer_handle->buffer_size;
-	auto part1_ptr = PreviousNewline(prev_buffer_ptr);
+	auto part1_ptr = PreviousNewline(prev_buffer_ptr, previous_buffer_handle->buffer_size);
 	auto part1_size = prev_buffer_ptr - part1_ptr;
 
 	// Now copy the data to our reconstruct buffer
 	const auto reconstruct_ptr = reconstruct_buffer.get();
 	memcpy(reconstruct_ptr, part1_ptr, part1_size);
-	// Now find the newline in the current block
-	auto line_end = NextNewline(buffer_ptr, buffer_size);
-	if (line_end == nullptr) {
-		ThrowObjectSizeError(buffer_size - buffer_offset);
-	} else {
-		line_end++;
+
+	// We copied the object, so we are no longer reading the previous buffer
+	if (--previous_buffer_handle->readers == 0) {
+		current_reader->RemoveBuffer(*previous_buffer_handle);
 	}
-	idx_t part2_size = line_end - buffer_ptr;
 
-	idx_t line_size = part1_size + part2_size;
-	if (line_size > bind_data.maximum_object_size) {
-		ThrowObjectSizeError(line_size);
+	if (part1_size == 1) {
+		// Just a newline
+		return false;
 	}
 
-	// And copy the remainder of the line to the reconstruct buffer
-	memcpy(reconstruct_ptr + part1_size, buffer_ptr, part2_size);
-	memset(reconstruct_ptr + line_size, 0, YYJSON_PADDING_SIZE);
-	buffer_offset += part2_size;
+	idx_t line_size = part1_size;
+	if (buffer_size != 0) {
+		// Now find the newline in the current block
+		auto line_end = NextNewline(buffer_ptr, buffer_size);
+		if (line_end == nullptr) {
+			ThrowObjectSizeError(buffer_size - buffer_offset);
+		} else {
+			line_end++;
+		}
+		idx_t part2_size = line_end - buffer_ptr;
 
-	// We copied the object, so we are no longer reading the previous buffer
-	if (--previous_buffer_handle->readers == 0) {
-		current_reader->RemoveBuffer(current_buffer_handle->buffer_index - 1);
+		line_size += part2_size;
+		if (line_size > bind_data.maximum_object_size) {
+			ThrowObjectSizeError(line_size);
+		}
+
+		// And copy the remainder of the line to the reconstruct buffer
+		memcpy(reconstruct_ptr + part1_size, buffer_ptr, part2_size);
+		memset(reconstruct_ptr + line_size, 0, YYJSON_PADDING_SIZE);
+		buffer_offset += part2_size;
 	}
 
 	ParseJSON(char_ptr_cast(reconstruct_ptr), line_size, line_size);
+
+	return true;
 }
 
 void JSONScanLocalState::ParseNextChunk() {
@@ -846,7 +868,7 @@ void JSONScanLocalState::ParseNextChunk() {
 			if (!is_last) {
 				// Last bit of data belongs to the next batch
 				if (format != JSONFormat::NEWLINE_DELIMITED) {
-					if (scan_count == 0) {
+					if (remaining > bind_data.maximum_object_size) {
 						ThrowObjectSizeError(remaining);
 					}
 					memcpy(reconstruct_buffer.get(), json_start, remaining);