duckdb 0.7.2-dev3515.0 → 0.7.2-dev3666.0

package/src/duckdb/extension/json/json_functions/read_json.cpp

@@ -8,88 +8,83 @@ namespace duckdb {
 
 void JSONScan::AutoDetect(ClientContext &context, JSONScanData &bind_data, vector<LogicalType> &return_types,
                           vector<string> &names) {
-	auto original_scan_type = bind_data.type;
-	bind_data.type = JSONScanType::SAMPLE; // Set scan type to sample for the auto-detect, we restore it later
-	JSONScanGlobalState gstate(context, bind_data);
-	JSONScanLocalState lstate(context, gstate);
-	ArenaAllocator allocator(BufferAllocator::Get(context));
+	// Change scan type during detection
+	bind_data.type = JSONScanType::SAMPLE;
 
-	// Read for the specified sample size
+	// These are used across files (if union_by_name)
 	JSONStructureNode node;
-	bool more_than_one = false;
+	ArenaAllocator allocator(BufferAllocator::Get(context));
 	Vector string_vector(LogicalType::VARCHAR);
-	idx_t remaining = bind_data.sample_size;
-	while (remaining != 0) {
-		allocator.Reset();
-
-		if (gstate.file_index >= 10) {
-			// We really shouldn't open more than 10 files when sampling
-			break;
-		}
 
-		auto read_count = lstate.ReadNext(gstate);
-		if (lstate.scan_count > 1) {
-			more_than_one = true;
-		}
-		if (read_count == 0) {
-			break;
-		}
-		idx_t next = MinValue<idx_t>(read_count, remaining);
-		yyjson_val **values;
-		if (bind_data.record_type == JSONRecordType::ARRAY_OF_RECORDS ||
-		    bind_data.record_type == JSONRecordType::ARRAY_OF_JSON) {
-			values = lstate.array_values;
+	// Loop through the files (if union_by_name, else just sample the first file)
+	for (idx_t file_idx = 0; file_idx < bind_data.files.size(); file_idx++) {
+		// Create global/local state and place the reader in the right field
+		JSONScanGlobalState gstate(context, bind_data);
+		JSONScanLocalState lstate(context, gstate);
+		if (file_idx == 0) {
+			gstate.json_readers.emplace_back(bind_data.initial_reader.get());
 		} else {
-			values = lstate.values;
+			gstate.json_readers.emplace_back(bind_data.union_readers[file_idx - 1].get());
 		}
-		for (idx_t i = 0; i < next; i++) {
-			if (values[i]) {
-				JSONStructure::ExtractStructure(values[i], node);
+
+		// Read and detect schema
+		idx_t remaining = bind_data.sample_size;
+		while (remaining != 0) {
+			allocator.Reset();
+			auto read_count = lstate.ReadNext(gstate);
+			if (read_count == 0) {
+				break;
+			}
+
+			idx_t next = MinValue<idx_t>(read_count, remaining);
+			for (idx_t i = 0; i < next; i++) {
+				const auto &val = lstate.values[i];
+				if (val) {
+					JSONStructure::ExtractStructure(val, node);
+				}
 			}
+			if (!node.ContainsVarchar()) { // Can't refine non-VARCHAR types
+				continue;
+			}
+			node.InitializeCandidateTypes(bind_data.max_depth);
+			node.RefineCandidateTypes(lstate.values, next, string_vector, allocator, bind_data.date_format_map);
+			remaining -= next;
+		}
+
+		if (file_idx == 0 && lstate.total_tuple_count != 0) {
+			bind_data.avg_tuple_size = lstate.total_read_size / lstate.total_tuple_count;
 		}
-		if (!node.ContainsVarchar()) { // Can't refine non-VARCHAR types
-			continue;
+
+		// Close the file and stop detection if not union_by_name
+		if (!bind_data.options.file_options.union_by_name) {
+			break;
 		}
-		node.InitializeCandidateTypes(bind_data.max_depth);
-		node.RefineCandidateTypes(values, next, string_vector, allocator, bind_data.date_format_map);
-		remaining -= next;
 	}
-	bind_data.type = original_scan_type;
+
+	// Restore the scan type
+	bind_data.type = JSONScanType::READ_JSON;
 
 	// Convert structure to logical type
 	auto type = JSONStructure::StructureToType(context, node, bind_data.max_depth);
 
-	// Detect record type
-	if (bind_data.record_type == JSONRecordType::AUTO) {
-		switch (type.id()) {
-		case LogicalTypeId::STRUCT:
-			bind_data.record_type = JSONRecordType::RECORDS;
-			break;
-		case LogicalTypeId::LIST: {
-			if (more_than_one) {
-				bind_data.record_type = JSONRecordType::JSON;
-			} else {
-				type = ListType::GetChildType(type);
-				if (type.id() == LogicalTypeId::STRUCT) {
-					bind_data.record_type = JSONRecordType::ARRAY_OF_RECORDS;
-				} else {
-					bind_data.record_type = JSONRecordType::ARRAY_OF_JSON;
-				}
-			}
-			break;
-		}
-		default:
-			bind_data.record_type = JSONRecordType::JSON;
+	// Auto-detect record type
+	if (bind_data.options.record_type == JSONRecordType::AUTO_DETECT) {
+		if (type.id() == LogicalTypeId::STRUCT) {
+			bind_data.options.record_type = JSONRecordType::RECORDS;
+		} else {
+			bind_data.options.record_type = JSONRecordType::VALUES;
 		}
 	}
 
-	// Detect return type
-	if (bind_data.auto_detect) {
-		bind_data.transform_options.date_format_map = &bind_data.date_format_map;
-		if (type.id() != LogicalTypeId::STRUCT) {
-			return_types.emplace_back(type);
-			names.emplace_back("json");
-		} else {
+	if (!bind_data.auto_detect) {
+		return;
+	}
+
+	bind_data.transform_options.date_format_map = &bind_data.date_format_map;
+
+	// Auto-detect columns
+	if (bind_data.options.record_type == JSONRecordType::RECORDS) {
+		if (type.id() == LogicalTypeId::STRUCT) {
 			const auto &child_types = StructType::GetChildTypes(type);
 			return_types.reserve(child_types.size());
 			names.reserve(child_types.size());
@@ -97,26 +92,29 @@ void JSONScan::AutoDetect(ClientContext &context, JSONScanData &bind_data, vecto
 				return_types.emplace_back(child_type.second);
 				names.emplace_back(child_type.first);
 			}
+		} else {
+			throw BinderException("json_read expected records, but got non-record JSON instead."
+			                      "\n Try setting records='auto' or records='false'.");
 		}
+	} else {
+		D_ASSERT(bind_data.options.record_type == JSONRecordType::VALUES);
+		return_types.emplace_back(type);
+		names.emplace_back("json");
 	}
-
-	for (auto &reader : gstate.json_readers) {
-		if (reader->IsOpen()) {
-			reader->Reset();
-		}
-	}
-	bind_data.stored_readers = std::move(gstate.json_readers);
 }
 
-void JSONScan::InitializeBindData(ClientContext &context, JSONScanData &bind_data,
-                                  const named_parameter_map_t &named_parameters, vector<string> &names,
-                                  vector<LogicalType> &return_types) {
-	for (auto &kv : named_parameters) {
+unique_ptr<FunctionData> ReadJSONBind(ClientContext &context, TableFunctionBindInput &input,
+                                      vector<LogicalType> &return_types, vector<string> &names) {
+	// First bind default params
+	auto bind_data = make_uniq<JSONScanData>();
+	bind_data->Bind(context, input);
+
+	for (auto &kv : input.named_parameters) {
 		auto loption = StringUtil::Lower(kv.first);
 		if (loption == "columns") {
 			auto &child_type = kv.second.type();
 			if (child_type.id() != LogicalTypeId::STRUCT) {
-				throw BinderException("read_json \"columns\" parameter requires a struct as input");
+				throw BinderException("read_json \"columns\" parameter requires a struct as input.");
 			}
 			auto &struct_children = StructValue::GetChildren(kv.second);
 			D_ASSERT(StructType::GetChildCount(child_type) == struct_children.size());
@@ -125,157 +123,158 @@ void JSONScan::InitializeBindData(ClientContext &context, JSONScanData &bind_dat
 				auto &val = struct_children[i];
 				names.push_back(name);
 				if (val.type().id() != LogicalTypeId::VARCHAR) {
-					throw BinderException("read_json \"columns\" parameter type specification must be VARCHAR");
+					throw BinderException("read_json \"columns\" parameter type specification must be VARCHAR.");
 				}
 				return_types.emplace_back(TransformStringToLogicalType(StringValue::Get(val), context));
 			}
 			D_ASSERT(names.size() == return_types.size());
 			if (names.empty()) {
-				throw BinderException("read_json \"columns\" parameter needs at least one column");
+				throw BinderException("read_json \"columns\" parameter needs at least one column.");
 			}
-			bind_data.names = names;
+			bind_data->names = names;
 		} else if (loption == "auto_detect") {
-			bind_data.auto_detect = BooleanValue::Get(kv.second);
+			bind_data->auto_detect = BooleanValue::Get(kv.second);
 		} else if (loption == "sample_size") {
 			auto arg = BigIntValue::Get(kv.second);
 			if (arg == -1) {
-				bind_data.sample_size = NumericLimits<idx_t>::Maximum();
+				bind_data->sample_size = NumericLimits<idx_t>::Maximum();
 			} else if (arg > 0) {
-				bind_data.sample_size = arg;
+				bind_data->sample_size = arg;
 			} else {
 				throw BinderException(
-				    "read_json \"sample_size\" parameter must be positive, or -1 to sample the entire file");
+				    "read_json \"sample_size\" parameter must be positive, or -1 to sample the entire file.");
 			}
 		} else if (loption == "maximum_depth") {
 			auto arg = BigIntValue::Get(kv.second);
 			if (arg == -1) {
-				bind_data.max_depth = NumericLimits<idx_t>::Maximum();
+				bind_data->max_depth = NumericLimits<idx_t>::Maximum();
 			} else {
-				bind_data.max_depth = arg;
+				bind_data->max_depth = arg;
 			}
 		} else if (loption == "dateformat" || loption == "date_format") {
 			auto format_string = StringValue::Get(kv.second);
 			if (StringUtil::Lower(format_string) == "iso") {
 				format_string = "%Y-%m-%d";
 			}
-			bind_data.date_format = format_string;
+			bind_data->date_format = format_string;
 
 			StrpTimeFormat format;
 			auto error = StrTimeFormat::ParseFormatSpecifier(format_string, format);
 			if (!error.empty()) {
-				throw InvalidInputException("Could not parse DATEFORMAT: %s", error.c_str());
+				throw InvalidInputException("read_json could not parse \"dateformat\": '%s'.", error.c_str());
 			}
 		} else if (loption == "timestampformat" || loption == "timestamp_format") {
 			auto format_string = StringValue::Get(kv.second);
 			if (StringUtil::Lower(format_string) == "iso") {
 				format_string = "%Y-%m-%dT%H:%M:%S.%fZ";
 			}
-			bind_data.timestamp_format = format_string;
+			bind_data->timestamp_format = format_string;
 
 			StrpTimeFormat format;
 			auto error = StrTimeFormat::ParseFormatSpecifier(format_string, format);
 			if (!error.empty()) {
-				throw InvalidInputException("Could not parse TIMESTAMPFORMAT: %s", error.c_str());
+				throw InvalidInputException("read_json could not parse \"timestampformat\": '%s'.", error.c_str());
 			}
-		} else if (loption == "json_format") {
+		} else if (loption == "records") {
 			auto arg = StringValue::Get(kv.second);
-			if (arg == "records") {
-				bind_data.record_type = JSONRecordType::RECORDS;
-			} else if (arg == "array_of_records") {
-				bind_data.record_type = JSONRecordType::ARRAY_OF_RECORDS;
-			} else if (arg == "values") {
-				bind_data.record_type = JSONRecordType::JSON;
-			} else if (arg == "array_of_values") {
-				bind_data.record_type = JSONRecordType::ARRAY_OF_JSON;
-			} else if (arg == "auto") {
-				bind_data.record_type = JSONRecordType::AUTO;
+			if (arg == "auto") {
+				bind_data->options.record_type = JSONRecordType::AUTO_DETECT;
+			} else if (arg == "true") {
+				bind_data->options.record_type = JSONRecordType::RECORDS;
+			} else if (arg == "false") {
+				bind_data->options.record_type = JSONRecordType::VALUES;
 			} else {
-				throw InvalidInputException("\"json_format\" must be one of ['records', 'array_of_records', 'json', "
-				                            "'array_of_json', 'auto']");
+				throw InvalidInputException("read_json requires \"records\" to be one of ['auto', 'true', 'false'].");
 			}
 		}
 	}
-}
-
-unique_ptr<FunctionData> ReadJSONBind(ClientContext &context, TableFunctionBindInput &input,
-                                      vector<LogicalType> &return_types, vector<string> &names) {
-	// First bind default params
-	auto result = JSONScanData::Bind(context, input);
-	auto &bind_data = (JSONScanData &)*result;
 
-	JSONScan::InitializeBindData(context, bind_data, input.named_parameters, names, return_types);
+	// Specifying column names overrides auto-detect
+	if (!return_types.empty()) {
+		bind_data->auto_detect = false;
+	}
 
-	if (!bind_data.names.empty()) {
-		bind_data.auto_detect = false; // override auto_detect when columns are specified
-	} else if (!bind_data.auto_detect) {
-		throw BinderException("read_json \"columns\" parameter is required when auto_detect is false");
+	if (!bind_data->auto_detect) {
+		// Need to specify columns if RECORDS and not auto-detecting
+		if (return_types.empty()) {
+			throw BinderException("read_json requires columns to be specified through the \"columns\" parameter."
+			                      "\n Use read_json_auto or set auto_detect=true to automatically guess columns.");
+		}
+		// If we are reading VALUES, we can only have one column
+		if (bind_data->options.record_type == JSONRecordType::VALUES && return_types.size() != 1) {
+			throw BinderException("read_json requires a single column to be specified through the \"columns\" "
+			                      "parameter when \"records\" is set to 'false'.");
+		}
 	}
 
-	bind_data.InitializeFormats();
+	bind_data->InitializeFormats();
 
-	if (bind_data.auto_detect || bind_data.record_type == JSONRecordType::AUTO) {
-		JSONScan::AutoDetect(context, bind_data, return_types, names);
-		bind_data.names = names;
+	if (bind_data->auto_detect || bind_data->options.record_type == JSONRecordType::AUTO_DETECT) {
+		JSONScan::AutoDetect(context, *bind_data, return_types, names);
+		bind_data->names = names;
+		D_ASSERT(return_types.size() == names.size());
 	}
 
-	auto &transform_options = bind_data.transform_options;
-	transform_options.strict_cast = !bind_data.ignore_errors;
-	transform_options.error_duplicate_key = !bind_data.ignore_errors;
+	bind_data->reader_bind =
+	    MultiFileReader::BindOptions(bind_data->options.file_options, bind_data->files, return_types, names);
+
+	auto &transform_options = bind_data->transform_options;
+	transform_options.strict_cast = !bind_data->ignore_errors;
+	transform_options.error_duplicate_key = !bind_data->ignore_errors;
 	transform_options.error_missing_key = false;
-	transform_options.error_unknown_key = bind_data.auto_detect && !bind_data.ignore_errors;
+	transform_options.error_unknown_key = bind_data->auto_detect && !bind_data->ignore_errors;
 	transform_options.delay_error = true;
 
-	return result;
+	return bind_data;
 }
 
 static void ReadJSONFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) {
-	auto &gstate = ((JSONGlobalTableFunctionState &)*data_p.global_state).state;
-	auto &lstate = ((JSONLocalTableFunctionState &)*data_p.local_state).state;
+	auto &gstate = data_p.global_state->Cast<JSONGlobalTableFunctionState>().state;
+	auto &lstate = data_p.local_state->Cast<JSONLocalTableFunctionState>().state;
 
 	const auto count = lstate.ReadNext(gstate);
-	yyjson_val **values;
-	if (gstate.bind_data.record_type == JSONRecordType::ARRAY_OF_RECORDS ||
-	    gstate.bind_data.record_type == JSONRecordType::ARRAY_OF_JSON) {
-		values = lstate.array_values;
-	} else {
-		D_ASSERT(gstate.bind_data.record_type != JSONRecordType::AUTO);
-		values = lstate.values;
-	}
+	yyjson_val **values = lstate.values;
 	output.SetCardinality(count);
 
-	vector<Vector *> result_vectors;
-	result_vectors.reserve(output.ColumnCount());
-	for (auto &valid_col_idx : gstate.bind_data.valid_cols) {
-		result_vectors.push_back(&output.data[valid_col_idx]);
-	}
-	D_ASSERT(result_vectors.size() == gstate.bind_data.names.size());
-
-	// Pass current reader to transform options so we can get line number information if an error occurs
-	bool success;
-	if (gstate.bind_data.record_type == JSONRecordType::RECORDS ||
-	    gstate.bind_data.record_type == JSONRecordType::ARRAY_OF_RECORDS) {
-		success = JSONTransform::TransformObject(values, lstate.GetAllocator(), count, gstate.bind_data.names,
-		                                         result_vectors, lstate.transform_options);
-	} else {
-		success = JSONTransform::Transform(values, lstate.GetAllocator(), *result_vectors[0], count,
-		                                   lstate.transform_options);
+	if (!gstate.names.empty()) {
+		vector<Vector *> result_vectors;
+		result_vectors.reserve(gstate.column_indices.size());
+		for (const auto &col_idx : gstate.column_indices) {
+			result_vectors.emplace_back(&output.data[col_idx]);
+		}
+
+		D_ASSERT(gstate.bind_data.options.record_type != JSONRecordType::AUTO_DETECT);
+		bool success;
+		if (gstate.bind_data.options.record_type == JSONRecordType::RECORDS) {
+			success = JSONTransform::TransformObject(values, lstate.GetAllocator(), count, gstate.names, result_vectors,
+			                                         lstate.transform_options);
+		} else {
+			D_ASSERT(gstate.bind_data.options.record_type == JSONRecordType::VALUES);
+			success = JSONTransform::Transform(values, lstate.GetAllocator(), *result_vectors[0], count,
+			                                   lstate.transform_options);
+		}
+
+		if (!success) {
+			string hint =
+			    gstate.bind_data.auto_detect
+			        ? "\nTry increasing 'sample_size', reducing 'maximum_depth', specifying 'columns', 'format' or "
+			          "'records' manually, or setting 'ignore_errors' to true."
+			        : "\nTry setting 'auto_detect' to true, specifying 'format' or 'records' manually, or setting "
+			          "'ignore_errors' to true.";
+			lstate.ThrowTransformError(lstate.transform_options.object_index,
+			                           lstate.transform_options.error_message + hint);
+		}
 	}
 
-	if (!success) {
-		string hint =
-		    gstate.bind_data.auto_detect
-		        ? "\nTry increasing 'sample_size', reducing 'maximum_depth', specifying 'columns', 'lines' or "
-		          "'json_format' manually, or setting 'ignore_errors' to true."
-		        : "\nTry setting 'auto_detect' to true, specifying 'lines' or 'json_format' manually, or setting "
-		          "'ignore_errors' to true.";
-		lstate.ThrowTransformError(lstate.transform_options.object_index,
-		                           lstate.transform_options.error_message + hint);
+	if (output.size() != 0) {
+		MultiFileReader::FinalizeChunk(gstate.bind_data.reader_bind, lstate.GetReaderData(), output);
 	}
 }
 
 TableFunction JSONFunctions::GetReadJSONTableFunction(shared_ptr<JSONScanInfo> function_info) {
 	TableFunction table_function({LogicalType::VARCHAR}, ReadJSONFunction, ReadJSONBind,
 	                             JSONGlobalTableFunctionState::Init, JSONLocalTableFunctionState::Init);
+	table_function.name = "read_json";
 
 	JSONScan::TableFunctionDefaults(table_function);
 	table_function.named_parameters["columns"] = LogicalType::ANY;
@@ -285,10 +284,9 @@ TableFunction JSONFunctions::GetReadJSONTableFunction(shared_ptr<JSONScanInfo> f
 	table_function.named_parameters["date_format"] = LogicalType::VARCHAR;
 	table_function.named_parameters["timestampformat"] = LogicalType::VARCHAR;
 	table_function.named_parameters["timestamp_format"] = LogicalType::VARCHAR;
-	table_function.named_parameters["json_format"] = LogicalType::VARCHAR;
+	table_function.named_parameters["records"] = LogicalType::VARCHAR;
 
-	table_function.projection_pushdown = true;
-	// TODO: might be able to do filter pushdown/prune too
+	// TODO: might be able to do filter pushdown/prune ?
 
 	table_function.function_info = std::move(function_info);
 
@@ -305,25 +303,25 @@ TableFunctionSet CreateJSONFunctionInfo(string name, shared_ptr<JSONScanInfo> in
 }
 
 TableFunctionSet JSONFunctions::GetReadJSONFunction() {
-	auto info =
-	    make_shared<JSONScanInfo>(JSONScanType::READ_JSON, JSONFormat::UNSTRUCTURED, JSONRecordType::RECORDS, false);
+	auto info = make_shared<JSONScanInfo>(JSONScanType::READ_JSON, JSONFormat::ARRAY, JSONRecordType::RECORDS);
 	return CreateJSONFunctionInfo("read_json", std::move(info));
 }
 
 TableFunctionSet JSONFunctions::GetReadNDJSONFunction() {
-	auto info = make_shared<JSONScanInfo>(JSONScanType::READ_JSON, JSONFormat::NEWLINE_DELIMITED,
-	                                      JSONRecordType::RECORDS, false);
+	auto info =
+	    make_shared<JSONScanInfo>(JSONScanType::READ_JSON, JSONFormat::NEWLINE_DELIMITED, JSONRecordType::RECORDS);
 	return CreateJSONFunctionInfo("read_ndjson", std::move(info));
 }
 
 TableFunctionSet JSONFunctions::GetReadJSONAutoFunction() {
-	auto info = make_shared<JSONScanInfo>(JSONScanType::READ_JSON, JSONFormat::AUTO_DETECT, JSONRecordType::AUTO, true);
+	auto info =
+	    make_shared<JSONScanInfo>(JSONScanType::READ_JSON, JSONFormat::AUTO_DETECT, JSONRecordType::AUTO_DETECT, true);
 	return CreateJSONFunctionInfo("read_json_auto", std::move(info), true);
 }
 
 TableFunctionSet JSONFunctions::GetReadNDJSONAutoFunction() {
-	auto info =
-	    make_shared<JSONScanInfo>(JSONScanType::READ_JSON, JSONFormat::NEWLINE_DELIMITED, JSONRecordType::AUTO, true);
+	auto info = make_shared<JSONScanInfo>(JSONScanType::READ_JSON, JSONFormat::NEWLINE_DELIMITED,
+	                                      JSONRecordType::AUTO_DETECT, true);
 	return CreateJSONFunctionInfo("read_ndjson_auto", std::move(info), true);
 }
 

package/src/duckdb/extension/json/json_functions/read_json_objects.cpp

@@ -6,34 +6,46 @@ namespace duckdb {
 
 unique_ptr<FunctionData> ReadJSONObjectsBind(ClientContext &context, TableFunctionBindInput &input,
                                              vector<LogicalType> &return_types, vector<string> &names) {
+	auto bind_data = make_uniq<JSONScanData>();
+	bind_data->Bind(context, input);
+
+	bind_data->names.emplace_back("json");
 	return_types.push_back(JSONCommon::JSONType());
 	names.emplace_back("json");
-	return JSONScanData::Bind(context, input);
+
+	bind_data->reader_bind =
+	    MultiFileReader::BindOptions(bind_data->options.file_options, bind_data->files, return_types, names);
+
+	return bind_data;
 }
 
 static void ReadJSONObjectsFunction(ClientContext &context, TableFunctionInput &data_p, DataChunk &output) {
-	D_ASSERT(output.ColumnCount() == 1);
-	D_ASSERT(JSONCommon::LogicalTypeIsJSON(output.data[0].GetType()));
-	auto &gstate = ((JSONGlobalTableFunctionState &)*data_p.global_state).state;
-	auto &lstate = ((JSONLocalTableFunctionState &)*data_p.local_state).state;
+	auto &gstate = data_p.global_state->Cast<JSONGlobalTableFunctionState>().state;
+	auto &lstate = data_p.local_state->Cast<JSONLocalTableFunctionState>().state;
 
 	// Fetch next lines
 	const auto count = lstate.ReadNext(gstate);
-	const auto lines = lstate.lines;
+	const auto units = lstate.units;
 	const auto objects = lstate.values;
 
-	// Create the strings without copying them
-	auto strings = FlatVector::GetData<string_t>(output.data[0]);
-	auto &validity = FlatVector::Validity(output.data[0]);
-	for (idx_t i = 0; i < count; i++) {
-		if (objects[i]) {
-			strings[i] = string_t(lines[i].pointer, lines[i].size);
-		} else {
-			validity.SetInvalid(i);
+	if (!gstate.names.empty()) {
+		// Create the strings without copying them
+		auto strings = FlatVector::GetData<string_t>(output.data[0]);
+		auto &validity = FlatVector::Validity(output.data[0]);
+		for (idx_t i = 0; i < count; i++) {
+			if (objects[i]) {
+				strings[i] = string_t(units[i].pointer, units[i].size);
+			} else {
+				validity.SetInvalid(i);
+			}
 		}
 	}
 
 	output.SetCardinality(count);
+
+	if (output.size() != 0) {
+		MultiFileReader::FinalizeChunk(gstate.bind_data.reader_bind, lstate.GetReaderData(), output);
+	}
 }
 
 TableFunction GetReadJSONObjectsTableFunction(bool list_parameter, shared_ptr<JSONScanInfo> function_info) {
@@ -49,7 +61,7 @@ TableFunction GetReadJSONObjectsTableFunction(bool list_parameter, shared_ptr<JS
 TableFunctionSet JSONFunctions::GetReadJSONObjectsFunction() {
 	TableFunctionSet function_set("read_json_objects");
 	auto function_info =
-	    make_shared<JSONScanInfo>(JSONScanType::READ_JSON_OBJECTS, JSONFormat::UNSTRUCTURED, JSONRecordType::JSON);
+	    make_shared<JSONScanInfo>(JSONScanType::READ_JSON_OBJECTS, JSONFormat::ARRAY, JSONRecordType::RECORDS);
 	function_set.AddFunction(GetReadJSONObjectsTableFunction(false, function_info));
 	function_set.AddFunction(GetReadJSONObjectsTableFunction(true, function_info));
 	return function_set;
@@ -57,8 +69,17 @@ TableFunctionSet JSONFunctions::GetReadJSONObjectsFunction() {
 
 TableFunctionSet JSONFunctions::GetReadNDJSONObjectsFunction() {
 	TableFunctionSet function_set("read_ndjson_objects");
+	auto function_info = make_shared<JSONScanInfo>(JSONScanType::READ_JSON_OBJECTS, JSONFormat::NEWLINE_DELIMITED,
+	                                               JSONRecordType::RECORDS);
+	function_set.AddFunction(GetReadJSONObjectsTableFunction(false, function_info));
+	function_set.AddFunction(GetReadJSONObjectsTableFunction(true, function_info));
+	return function_set;
+}
+
+TableFunctionSet JSONFunctions::GetReadJSONObjectsAutoFunction() {
+	TableFunctionSet function_set("read_json_objects_auto");
 	auto function_info =
-	    make_shared<JSONScanInfo>(JSONScanType::READ_JSON_OBJECTS, JSONFormat::NEWLINE_DELIMITED, JSONRecordType::JSON);
+	    make_shared<JSONScanInfo>(JSONScanType::READ_JSON_OBJECTS, JSONFormat::AUTO_DETECT, JSONRecordType::RECORDS);
 	function_set.AddFunction(GetReadJSONObjectsTableFunction(false, function_info));
 	function_set.AddFunction(GetReadJSONObjectsTableFunction(true, function_info));
 	return function_set;

package/src/duckdb/extension/json/json_functions.cpp

@@ -1,5 +1,6 @@
 #include "json_functions.hpp"
 
+#include "duckdb/common/file_system.hpp"
 #include "duckdb/execution/expression_executor.hpp"
 #include "duckdb/function/cast/cast_function_set.hpp"
 #include "duckdb/function/cast/default_casts.hpp"
@@ -50,7 +51,7 @@ bool JSONReadFunctionData::Equals(const FunctionData &other_p) const {
 }
 
 unique_ptr<FunctionData> JSONReadFunctionData::Bind(ClientContext &context, ScalarFunction &bound_function,
-                                                    vector<duckdb::unique_ptr<Expression>> &arguments) {
+                                                    vector<unique_ptr<Expression>> &arguments) {
 	D_ASSERT(bound_function.arguments.size() == 2);
 	bool constant = false;
 	string path = "";
@@ -80,7 +81,7 @@ bool JSONReadManyFunctionData::Equals(const FunctionData &other_p) const {
 }
 
 unique_ptr<FunctionData> JSONReadManyFunctionData::Bind(ClientContext &context, ScalarFunction &bound_function,
-                                                        vector<duckdb::unique_ptr<Expression>> &arguments) {
+                                                        vector<unique_ptr<Expression>> &arguments) {
 	D_ASSERT(bound_function.arguments.size() == 2);
 	if (arguments[1]->HasParameter()) {
 		throw ParameterNotResolvedException();
@@ -173,6 +174,7 @@ vector<TableFunctionSet> JSONFunctions::GetTableFunctions() {
 	// Reads JSON as string
 	functions.push_back(GetReadJSONObjectsFunction());
 	functions.push_back(GetReadNDJSONObjectsFunction());
+	functions.push_back(GetReadJSONObjectsAutoFunction());
 
 	// Read JSON as columnar data
 	functions.push_back(GetReadJSONFunction());
@@ -199,16 +201,21 @@ unique_ptr<TableRef> JSONFunctions::ReadJSONReplacement(ClientContext &context,
 		return nullptr;
 	}
 	auto table_function = make_uniq<TableFunctionRef>();
-	vector<duckdb::unique_ptr<ParsedExpression>> children;
+	vector<unique_ptr<ParsedExpression>> children;
 	children.push_back(make_uniq<ConstantExpression>(Value(table_name)));
 	table_function->function = make_uniq<FunctionExpression>("read_json_auto", std::move(children));
+
+	if (!FileSystem::HasGlob(table_name)) {
+		table_function->alias = FileSystem::ExtractBaseName(table_name);
+	}
+
 	return std::move(table_function);
 }
 
 static bool CastVarcharToJSON(Vector &source, Vector &result, idx_t count, CastParameters &parameters) {
 	auto &lstate = parameters.local_state->Cast<JSONFunctionLocalState>();
 	lstate.json_allocator.Reset();
-	auto alc = lstate.json_allocator.GetYYJSONAllocator();
+	auto alc = lstate.json_allocator.GetYYAlc();
 
 	bool success = true;
 	UnaryExecutor::ExecuteWithNulls<string_t, string_t>(