duckdb 0.6.2-dev1978.0 → 0.6.2-dev2015.0

package/package.json

@@ -2,7 +2,7 @@
   "name": "duckdb",
   "main": "./lib/duckdb.js",
   "types": "./lib/duckdb.d.ts",
-  "version": "0.6.2-dev1978.0",
+  "version": "0.6.2-dev2015.0",
   "description": "DuckDB node.js API",
   "gypfile": true,
   "dependencies": {

package/src/duckdb/extension/json/buffered_json_reader.cpp

@@ -2,7 +2,6 @@
 
 #include "duckdb/common/field_writer.hpp"
 #include "duckdb/common/file_opener.hpp"
-#include "duckdb/common/file_system.hpp"
 #include "duckdb/common/printer.hpp"
 
 namespace duckdb {
@@ -20,13 +19,13 @@ void BufferedJSONReaderOptions::Deserialize(FieldReader &reader) {
 }
 
 JSONBufferHandle::JSONBufferHandle(idx_t buffer_index_p, idx_t readers_p, AllocatedData &&buffer_p, idx_t buffer_size_p)
-    : buffer_index(buffer_index_p), readers(readers_p), buffer(move(buffer_p)), buffer_size(buffer_size_p) {
+    : buffer_index(buffer_index_p), readers(readers_p), buffer(std::move(buffer_p)), buffer_size(buffer_size_p) {
 }
 
-JSONFileHandle::JSONFileHandle(unique_ptr<FileHandle> file_handle_p)
-    : file_handle(move(file_handle_p)), can_seek(file_handle->CanSeek()),
-      plain_file_source(file_handle->OnDiskFile() && can_seek), file_size(file_handle->GetFileSize()),
-      read_position(0) {
+JSONFileHandle::JSONFileHandle(unique_ptr<FileHandle> file_handle_p, Allocator &allocator_p)
+    : file_handle(std::move(file_handle_p)), allocator(allocator_p), can_seek(file_handle->CanSeek()),
+      plain_file_source(file_handle->OnDiskFile() && can_seek), file_size(file_handle->GetFileSize()), read_position(0),
+      cached_size(0) {
 }
 
 idx_t JSONFileHandle::FileSize() const {
@@ -37,12 +36,16 @@ idx_t JSONFileHandle::Remaining() const {
 	return file_size - read_position;
 }
 
+bool JSONFileHandle::PlainFileSource() const {
+	return plain_file_source;
+}
+
 bool JSONFileHandle::CanSeek() const {
 	return can_seek;
 }
 
-bool JSONFileHandle::PlainFileSource() const {
-	return plain_file_source;
+void JSONFileHandle::Seek(idx_t position) {
+	file_handle->Seek(position);
 }
 
 idx_t JSONFileHandle::GetPositionAndSize(idx_t &position, idx_t requested_size) {
@@ -53,22 +56,82 @@ idx_t JSONFileHandle::GetPositionAndSize(idx_t &position, idx_t requested_size)
 	return actual_size;
 }
 
-void JSONFileHandle::ReadAtPosition(const char *pointer, idx_t size, idx_t position) {
+void JSONFileHandle::ReadAtPosition(const char *pointer, idx_t size, idx_t position, bool sample_run) {
 	D_ASSERT(size != 0);
-	file_handle->Read((void *)pointer, size, position);
+	if (plain_file_source) {
+		file_handle->Read((void *)pointer, size, position);
+		return;
+	}
+
+	if (sample_run) { // Cache the buffer
+		file_handle->Read((void *)pointer, size, position);
+		cached_buffers.emplace_back(allocator.Allocate(size));
+		memcpy(cached_buffers.back().get(), pointer, size);
+		cached_size += size;
+		return;
+	}
+
+	if (!cached_buffers.empty() || position < cached_size) {
+		ReadFromCache(pointer, size, position);
+	}
+	if (size != 0) {
+		file_handle->Read((void *)pointer, size, position);
+	}
 }
 
-idx_t JSONFileHandle::Read(const char *pointer, idx_t requested_size) {
+idx_t JSONFileHandle::Read(const char *pointer, idx_t requested_size, bool sample_run) {
 	D_ASSERT(requested_size != 0);
-	auto actual_size = file_handle->Read((void *)pointer, requested_size);
-	read_position += actual_size;
+	if (plain_file_source) {
+		auto actual_size = file_handle->Read((void *)pointer, requested_size);
+		read_position += actual_size;
+		return actual_size;
+	}
+
+	if (sample_run) { // Cache the buffer
+		auto actual_size = file_handle->Read((void *)pointer, requested_size);
+		if (actual_size > 0) {
+			cached_buffers.emplace_back(allocator.Allocate(actual_size));
+			memcpy(cached_buffers.back().get(), pointer, actual_size);
+		}
+		cached_size += actual_size;
+		read_position += actual_size;
+		return actual_size;
+	}
+
+	idx_t actual_size = 0;
+	if (!cached_buffers.empty() || read_position < cached_size) {
+		actual_size += ReadFromCache(pointer, requested_size, read_position);
+	}
+	if (requested_size != 0) {
+		actual_size += file_handle->Read((void *)pointer, requested_size);
+	}
 	return actual_size;
 }
 
-BufferedJSONReader::BufferedJSONReader(ClientContext &context, BufferedJSONReaderOptions options_p, idx_t file_index_p,
-                                       string file_path_p)
-    : file_index(file_index_p), file_path(std::move(file_path_p)), context(context), options(std::move(options_p)),
-      buffer_index(0) {
+idx_t JSONFileHandle::ReadFromCache(const char *&pointer, idx_t &size, idx_t &position) {
+	idx_t read_size = 0;
+	idx_t total_offset = 0;
+	for (auto &cached_buffer : cached_buffers) {
+		if (size == 0) {
+			break;
+		}
+		if (position < total_offset + cached_buffer.GetSize()) {
+			idx_t within_buffer_offset = position - total_offset;
+			idx_t copy_size = MinValue<idx_t>(size, cached_buffer.GetSize() - within_buffer_offset);
+			memcpy((void *)pointer, cached_buffer.get() + within_buffer_offset, copy_size);
+
+			read_size += copy_size;
+			pointer += copy_size;
+			size -= copy_size;
+			position += copy_size;
+		}
+		total_offset += cached_buffer.GetSize();
+	}
+	return read_size;
+}
+
+BufferedJSONReader::BufferedJSONReader(ClientContext &context, BufferedJSONReaderOptions options_p, string file_path_p)
+    : file_path(std::move(file_path_p)), context(context), options(std::move(options_p)), buffer_index(0) {
 }
 
 void BufferedJSONReader::OpenJSONFile() {
@@ -77,7 +140,7 @@ void BufferedJSONReader::OpenJSONFile() {
 	auto file_opener = FileOpener::Get(context);
 	auto regular_file_handle = file_system.OpenFile(file_path.c_str(), FileFlags::FILE_FLAGS_READ,
 	                                                FileLockType::NO_LOCK, options.compression, file_opener);
-	file_handle = make_unique<JSONFileHandle>(std::move(regular_file_handle));
+	file_handle = make_unique<JSONFileHandle>(std::move(regular_file_handle), BufferAllocator::Get(context));
 }
 
 bool BufferedJSONReader::IsOpen() {
@@ -113,9 +176,40 @@ AllocatedData BufferedJSONReader::RemoveBuffer(idx_t buffer_idx) {
 }
 
 idx_t BufferedJSONReader::GetBufferIndex() {
+	buffer_line_or_object_counts.push_back(-1);
 	return buffer_index++;
 }
 
+void BufferedJSONReader::SetBufferLineOrObjectCount(idx_t index, idx_t count) {
+	lock_guard<mutex> guard(lock);
+	buffer_line_or_object_counts[index] = count;
+}
+
+void BufferedJSONReader::ThrowParseError(idx_t buf_index, idx_t line_or_object_in_buf, yyjson_read_err &err,
+                                         const string &extra) {
+	D_ASSERT(options.format == JSONFormat::UNSTRUCTURED || options.format == JSONFormat::NEWLINE_DELIMITED);
+	while (true) {
+		lock_guard<mutex> guard(lock);
+		idx_t line = line_or_object_in_buf;
+		bool can_throw = true;
+		for (idx_t b_idx = 0; b_idx < buf_index; b_idx++) {
+			if (buffer_line_or_object_counts[b_idx] == -1) {
+				can_throw = false;
+				break;
+			} else {
+				line += buffer_line_or_object_counts[b_idx];
+			}
+		}
+		if (!can_throw) {
+			continue;
+		}
+		string unit = options.format == JSONFormat::NEWLINE_DELIMITED ? "line" : "object";
+		// SQL uses 1-based indexing so I guess we will do that in our exception here as well
+		throw InvalidInputException("Malformed JSON in file \"%s\", at byte %llu in %s %llu: %s. %s", file_path,
+		                            err.pos + 1, unit, line + 1, err.msg, extra);
+	}
+}
+
 double BufferedJSONReader::GetProgress() const {
 	if (file_handle) {
 		return 100.0 * double(file_handle->Remaining()) / double(file_handle->FileSize());
@@ -124,4 +218,24 @@ double BufferedJSONReader::GetProgress() const {
 	}
 }
 
+void BufferedJSONReader::Reset() {
+	buffer_index = 0;
+	buffer_map.clear();
+	buffer_line_or_object_counts.clear();
+
+	if (file_handle->CanSeek()) {
+		file_handle->Seek(0);
+	} else {
+		file_handle->Reset();
+	}
+	file_handle->Reset();
+}
+
+void JSONFileHandle::Reset() {
+	read_position = 0;
+	if (plain_file_source) {
+		file_handle->Reset();
+	}
+}
+
 } // namespace duckdb

package/src/duckdb/extension/json/include/buffered_json_reader.hpp

@@ -10,13 +10,12 @@
 
 #include "duckdb/common/atomic.hpp"
 #include "duckdb/common/enums/file_compression_type.hpp"
+#include "duckdb/common/file_system.hpp"
 #include "duckdb/common/mutex.hpp"
 #include "json_common.hpp"
 
 namespace duckdb {
 
-struct FileHandle;
-
 enum class JSONFormat : uint8_t {
 	//! Auto-detect format (UNSTRUCTURED / NEWLINE_DELIMITED)
 	AUTO_DETECT = 0,
@@ -58,21 +57,28 @@ public:
 
 struct JSONFileHandle {
 public:
-	explicit JSONFileHandle(unique_ptr<FileHandle> file_handle);
+	JSONFileHandle(unique_ptr<FileHandle> file_handle, Allocator &allocator);
 
 	idx_t FileSize() const;
 	idx_t Remaining() const;
 
-	bool CanSeek() const;
 	bool PlainFileSource() const;
+	bool CanSeek() const;
+	void Seek(idx_t position);
 
 	idx_t GetPositionAndSize(idx_t &position, idx_t requested_size);
-	void ReadAtPosition(const char *pointer, idx_t size, idx_t position);
-	idx_t Read(const char *pointer, idx_t requested_size);
+	void ReadAtPosition(const char *pointer, idx_t size, idx_t position, bool sample_run);
+	idx_t Read(const char *pointer, idx_t requested_size, bool sample_run);
+
+	void Reset();
+
+private:
+	idx_t ReadFromCache(const char *&pointer, idx_t &size, idx_t &position);
 
 private:
 	//! The JSON file handle
 	unique_ptr<FileHandle> file_handle;
+	Allocator &allocator;
 
 	//! File properties
 	const bool can_seek;
@@ -81,11 +87,15 @@ private:
 
 	//! Read properties
 	idx_t read_position;
+
+	//! Cached buffers for resetting when reading stream
+	vector<AllocatedData> cached_buffers;
+	idx_t cached_size;
 };
 
 class BufferedJSONReader {
 public:
-	BufferedJSONReader(ClientContext &context, BufferedJSONReaderOptions options, idx_t file_index, string file_path);
+	BufferedJSONReader(ClientContext &context, BufferedJSONReaderOptions options, string file_path);
 
 	void OpenJSONFile();
 	bool IsOpen();
@@ -93,18 +103,25 @@ public:
 	BufferedJSONReaderOptions &GetOptions();
 	JSONFileHandle &GetFileHandle() const;
 
+	//! Insert/get/remove buffer (grabs the lock)
 	void InsertBuffer(idx_t buffer_idx, unique_ptr<JSONBufferHandle> &&buffer);
 	JSONBufferHandle *GetBuffer(idx_t buffer_idx);
 	AllocatedData RemoveBuffer(idx_t buffer_idx);
+
+	//! Get a new buffer index (must hold the lock)
 	idx_t GetBufferIndex();
+	//! Set line count for a buffer that is done (grabs the lock)
+	void SetBufferLineOrObjectCount(idx_t index, idx_t count);
+	//! Throws an error that mentions the file name and line number
+	void ThrowParseError(idx_t buf_index, idx_t line_or_object_in_buf, yyjson_read_err &err, const string &extra = "");
 
 	double GetProgress() const;
+	void Reset();
 
 public:
 	mutex lock;
 
-	//! File index / path
-	const idx_t file_index;
+	//! File path
 	const string file_path;
 
 private:
@@ -118,6 +135,9 @@ private:
 	idx_t buffer_index;
 	//! Mapping from batch index to currently held buffers
 	unordered_map<idx_t, unique_ptr<JSONBufferHandle>> buffer_map;
+
+	//! Line count per buffer
+	vector<int64_t> buffer_line_or_object_counts;
 };
 
 } // namespace duckdb

package/src/duckdb/extension/json/include/json_common.hpp

@@ -50,6 +50,37 @@ private:
 	yyjson_alc yyjson_allocator;
 };
 
+struct JSONKey {
+	const char *ptr;
+	size_t len;
+};
+
+struct JSONKeyHash {
+	inline std::size_t operator()(const JSONKey &k) const {
+		size_t result;
+		if (k.len >= sizeof(size_t)) {
+			memcpy(&result, k.ptr + k.len - sizeof(size_t), sizeof(size_t));
+		} else {
+			result = 0;
+			duckdb::FastMemcpy(&result, k.ptr, k.len);
+		}
+		return result;
+	}
+};
+
+struct JSONKeyEquality {
+	inline bool operator()(const JSONKey &a, const JSONKey &b) const {
+		if (a.len != b.len) {
+			return false;
+		}
+		return duckdb::FastMemcmp(a.ptr, b.ptr, a.len) == 0;
+	}
+};
+
+template <typename T>
+using json_key_map_t = unordered_map<JSONKey, T, JSONKeyHash, JSONKeyEquality>;
+using json_key_set_t = unordered_set<JSONKey, JSONKeyHash, JSONKeyEquality>;
+
 struct JSONCommon {
 public:
 	static constexpr auto JSON_TYPE_NAME = "JSON";
@@ -111,6 +142,31 @@ public:
 		return string_t(ValTypeToString<YYJSON_VAL_T>(val));
 	}
 
+	template <class YYJSON_VAL_T>
+	static inline const LogicalTypeId ValTypeToLogicalTypeId(YYJSON_VAL_T *val) {
+		switch (GetTag<YYJSON_VAL_T>(val)) {
+		case YYJSON_TYPE_NULL | YYJSON_SUBTYPE_NONE:
+			return LogicalTypeId::SQLNULL;
+		case YYJSON_TYPE_STR | YYJSON_SUBTYPE_NONE:
+			return LogicalTypeId::VARCHAR;
+		case YYJSON_TYPE_ARR | YYJSON_SUBTYPE_NONE:
+			return LogicalTypeId::LIST;
+		case YYJSON_TYPE_OBJ | YYJSON_SUBTYPE_NONE:
+			return LogicalTypeId::STRUCT;
+		case YYJSON_TYPE_BOOL | YYJSON_SUBTYPE_TRUE:
+		case YYJSON_TYPE_BOOL | YYJSON_SUBTYPE_FALSE:
+			return LogicalTypeId::BOOLEAN;
+		case YYJSON_TYPE_NUM | YYJSON_SUBTYPE_UINT:
+			return LogicalTypeId::UBIGINT;
+		case YYJSON_TYPE_NUM | YYJSON_SUBTYPE_SINT:
+			return LogicalTypeId::BIGINT;
+		case YYJSON_TYPE_NUM | YYJSON_SUBTYPE_REAL:
+			return LogicalTypeId::DOUBLE;
+		default:
+			throw InternalException("Unexpected yyjson tag in ValTypeToLogicalTypeId");
+		}
+	}
+
 public:
 	static inline yyjson_mut_doc *CreateDocument(yyjson_alc *alc) {
 		D_ASSERT(alc);

package/src/duckdb/extension/json/include/json_functions.hpp

@@ -14,6 +14,8 @@
 
 namespace duckdb {
 
+class TableRef;
+struct ReplacementScanData;
 class CastFunctionSet;
 struct CastParameters;
 
@@ -62,6 +64,8 @@ class JSONFunctions {
 public:
 	static vector<CreateScalarFunctionInfo> GetScalarFunctions();
 	static vector<CreateTableFunctionInfo> GetTableFunctions();
+	static unique_ptr<TableRef> ReadJSONReplacement(ClientContext &context, const string &table_name,
+	                                                ReplacementScanData *data);
 	static void RegisterCastFunctions(CastFunctionSet &casts);
 
 private:
@@ -82,6 +86,7 @@ private:
 
 	static CreateScalarFunctionInfo GetArrayLengthFunction();
 	static CreateScalarFunctionInfo GetContainsFunction();
+	static CreateScalarFunctionInfo GetKeysFunction();
 	static CreateScalarFunctionInfo GetTypeFunction();
 	static CreateScalarFunctionInfo GetValidFunction();
 
@@ -97,6 +102,10 @@ private:
 	// Table functions
 	static CreateTableFunctionInfo GetReadJSONObjectsFunction();
 	static CreateTableFunctionInfo GetReadNDJSONObjectsFunction();
+	static CreateTableFunctionInfo GetReadJSONFunction();
+	static CreateTableFunctionInfo GetReadNDJSONFunction();
+	static CreateTableFunctionInfo GetReadJSONAutoFunction();
+	static CreateTableFunctionInfo GetReadNDJSONAutoFunction();
 };
 
 } // namespace duckdb

package/src/duckdb/extension/json/include/json_scan.hpp

@@ -10,11 +10,59 @@
 
 #include "buffered_json_reader.hpp"
 #include "duckdb/common/mutex.hpp"
+#include "duckdb/function/scalar/strftime.hpp"
 #include "duckdb/function/table_function.hpp"
+#include "json_transform.hpp"
 
 namespace duckdb {
 
-struct JSONScanLocalState;
+enum class JSONScanType : uint8_t {
+	INVALID = 0,
+	//! Read JSON straight to columnar data
+	READ_JSON = 1,
+	//! Read JSON objects as strings
+	READ_JSON_OBJECTS = 2,
+	//! Sample run for schema detection
+	SAMPLE = 3,
+};
+
+//! Even though LogicalTypeId is just a uint8_t, this is still needed ...
+struct LogicalTypeIdHash {
+	inline std::size_t operator()(const LogicalTypeId &id) const {
+		return (size_t)id;
+	}
+};
+
+struct DateFormatMap {
+public:
+	void Initialize(const unordered_map<LogicalTypeId, vector<const char *>, LogicalTypeIdHash> &format_templates) {
+		for (const auto &entry : format_templates) {
+			auto &formats = candidate_formats.emplace(entry.first, vector<StrpTimeFormat>()).first->second;
+			formats.reserve(entry.second.size());
+			for (const auto &format : entry.second) {
+				formats.emplace_back();
+				formats.back().format_specifier = format;
+				StrpTimeFormat::ParseFormatSpecifier(formats.back().format_specifier, formats.back());
+			}
+		}
+	}
+
+	bool HasFormats(LogicalTypeId type) const {
+		return candidate_formats.find(type) != candidate_formats.end();
+	}
+
+	vector<StrpTimeFormat> &GetCandidateFormats(LogicalTypeId type) {
+		D_ASSERT(HasFormats(type));
+		return candidate_formats[type];
+	}
+
+	StrpTimeFormat &GetFormat(LogicalTypeId type) {
+		return candidate_formats[type].back();
+	}
+
+private:
+	unordered_map<LogicalTypeId, vector<StrpTimeFormat>, LogicalTypeIdHash> candidate_formats;
+};
 
 struct JSONScanData : public TableFunctionData {
 public:
@@ -27,6 +75,8 @@ public:
 	void Deserialize(FieldReader &reader);
 
 public:
+	//! Scan type
+	JSONScanType type;
 	//! File-specific options
 	BufferedJSONReaderOptions options;
 	//! The files we're reading
@@ -34,27 +84,41 @@ public:
 
 	//! Whether or not we should ignore malformed JSON (default to NULL)
 	bool ignore_errors = false;
-	//! Maximum JSON object size (defaults to 1MB)
+	//! Maximum JSON object size (defaults to 1MB minimum)
 	idx_t maximum_object_size = 1048576;
-	//! Whether we return JSON strings (if not, we return YYJSON documents)
-	bool return_json_strings = true;
+	//! Options when transforming the JSON to columnar data
+	JSONTransformOptions transform_options;
+
+	//! Whether we auto-detect a schema
+	bool auto_detect = false;
+	//! Sample size for detecting schema
+	idx_t sample_size = STANDARD_VECTOR_SIZE;
+	//! Column names (in order)
+	vector<string> names;
+	//! Max depth we go to detect nested JSON schema (defaults to unlimited)
+	idx_t max_depth = NumericLimits<idx_t>::Maximum();
+
+	//! Stored readers for when we're detecting the schema
+	vector<unique_ptr<BufferedJSONReader>> stored_readers;
+	//! Candidate date formats
+	DateFormatMap date_format_map;
 };
 
 struct JSONScanInfo : public TableFunctionInfo {
 public:
-	explicit JSONScanInfo(JSONFormat forced_format_p, bool return_strings)
-	    : forced_format(forced_format_p), return_json_strings(return_strings) {
+	explicit JSONScanInfo(JSONScanType type_p = JSONScanType::INVALID, JSONFormat format_p = JSONFormat::AUTO_DETECT,
+	                      bool auto_detect_p = false)
+	    : type(type_p), format(format_p), auto_detect(auto_detect_p) {
 	}
 
-	JSONFormat forced_format;
-	bool return_json_strings;
+	JSONScanType type;
+	JSONFormat format;
+	bool auto_detect;
 };
 
-struct JSONScanGlobalState : public GlobalTableFunctionState {
+struct JSONScanGlobalState {
 public:
 	JSONScanGlobalState(ClientContext &context, JSONScanData &bind_data);
-	static unique_ptr<GlobalTableFunctionState> Init(ClientContext &context, TableFunctionInitInput &input);
-	idx_t MaxThreads() const override;
 
 public:
 	//! Bound data
@@ -70,7 +134,7 @@ public:
 	vector<unique_ptr<BufferedJSONReader>> json_readers;
 	//! Current file/batch index
 	idx_t file_index;
-	idx_t batch_index;
+	atomic<idx_t> batch_index;
 
 	//! Current number of threads active
 	idx_t system_threads;
@@ -96,23 +160,25 @@ public:
 	}
 };
 
-struct JSONScanLocalState : public LocalTableFunctionState {
+struct JSONScanLocalState {
 public:
 	JSONScanLocalState(ClientContext &context, JSONScanGlobalState &gstate);
-	static unique_ptr<LocalTableFunctionState> Init(ExecutionContext &context, TableFunctionInitInput &input,
-	                                                GlobalTableFunctionState *global_state);
+
+public:
 	idx_t ReadNext(JSONScanGlobalState &gstate);
-	idx_t GetBatchIndex() const;
+	yyjson_alc *GetAllocator();
 
 	JSONLine lines[STANDARD_VECTOR_SIZE];
-	yyjson_doc *objects[STANDARD_VECTOR_SIZE];
+	yyjson_val *objects[STANDARD_VECTOR_SIZE];
 
 	idx_t batch_index;
 
 private:
-	yyjson_doc *ParseLine(char *line_start, idx_t line_size, JSONLine &line, const bool &ignore_errors);
+	yyjson_val *ParseLine(char *line_start, idx_t line_size, idx_t remaining, JSONLine &line);
 
 private:
+	//! Bind data
+	JSONScanData &bind_data;
 	//! Thread-local allocator
 	JSONAllocator json_allocator;
 
@@ -127,6 +193,7 @@ private:
 	idx_t buffer_size;
 	idx_t buffer_offset;
 	idx_t prev_buffer_remainder;
+	idx_t lines_or_objects_in_buffer;
 
 	//! Buffer to reconstruct split objects
 	AllocatedData reconstruct_buffer;
@@ -135,21 +202,43 @@ private:
 	const char *buffer_copy_ptr;
 
 private:
-	bool ReadNextBuffer(JSONScanGlobalState &gstate, bool &first_read);
-	void ReadNextBufferSeek(JSONScanGlobalState &gstate, bool &first_read, idx_t &buffer_index);
-	void ReadNextBufferNoSeek(JSONScanGlobalState &gstate, bool &first_read, idx_t &buffer_index);
+	bool ReadNextBuffer(JSONScanGlobalState &gstate);
+	void ReadNextBuffer(JSONScanGlobalState &gstate, idx_t &buffer_index);
+	void ReadNextBufferSeek(JSONScanGlobalState &gstate, idx_t &buffer_index);
+	void ReadNextBufferNoSeek(JSONScanGlobalState &gstate, idx_t &buffer_index);
 
 	void ReconstructFirstObject(JSONScanGlobalState &gstate);
 
 	void ReadUnstructured(idx_t &count);
-	void ReadNewlineDelimited(idx_t &count, const bool &ignore_errors);
+	void ReadNewlineDelimited(idx_t &count);
+};
+
+struct JSONGlobalTableFunctionState : public GlobalTableFunctionState {
+public:
+	JSONGlobalTableFunctionState(ClientContext &context, TableFunctionInitInput &input);
+	static unique_ptr<GlobalTableFunctionState> Init(ClientContext &context, TableFunctionInitInput &input);
+	idx_t MaxThreads() const override;
+
+public:
+	JSONScanGlobalState state;
+};
+
+struct JSONLocalTableFunctionState : public LocalTableFunctionState {
+public:
+	JSONLocalTableFunctionState(ClientContext &context, JSONScanGlobalState &gstate);
+	static unique_ptr<LocalTableFunctionState> Init(ExecutionContext &context, TableFunctionInitInput &input,
+	                                                GlobalTableFunctionState *global_state);
+	idx_t GetBatchIndex() const;
+
+public:
+	JSONScanLocalState state;
 };
 
 struct JSONScan {
 public:
 	static double JSONScanProgress(ClientContext &context, const FunctionData *bind_data_p,
 	                               const GlobalTableFunctionState *global_state) {
-		auto &gstate = (JSONScanGlobalState &)*global_state;
+		auto &gstate = ((JSONGlobalTableFunctionState &)*global_state).state;
 		double progress = 0;
 		for (auto &reader : gstate.json_readers) {
 			progress += reader->GetProgress();
@@ -159,14 +248,13 @@ public:
 
 	static idx_t JSONScanGetBatchIndex(ClientContext &context, const FunctionData *bind_data_p,
 	                                   LocalTableFunctionState *local_state, GlobalTableFunctionState *global_state) {
-		auto &lstate = (JSONScanLocalState &)*local_state;
+		auto &lstate = (JSONLocalTableFunctionState &)*local_state;
 		return lstate.GetBatchIndex();
 	}
 
 	static void JSONScanSerialize(FieldWriter &writer, const FunctionData *bind_data_p, const TableFunction &function) {
 		auto &bind_data = (JSONScanData &)*bind_data_p;
 		bind_data.Serialize(writer);
-		bind_data.options.Serialize(writer);
 	}
 
 	static unique_ptr<FunctionData> JSONScanDeserialize(ClientContext &context, FieldReader &reader,
@@ -180,6 +268,7 @@ public:
 		table_function.named_parameters["maximum_object_size"] = LogicalType::UINTEGER;
 		table_function.named_parameters["ignore_errors"] = LogicalType::BOOLEAN;
 		table_function.named_parameters["format"] = LogicalType::VARCHAR;
+		table_function.named_parameters["compression"] = LogicalType::VARCHAR;
 
 		table_function.table_scan_progress = JSONScanProgress;
 		table_function.get_batch_index = JSONScanGetBatchIndex;
@@ -187,6 +276,7 @@ public:
 		table_function.serialize = JSONScanSerialize;
 		table_function.deserialize = JSONScanDeserialize;
 
+		// TODO: might be able to do some of these
 		table_function.projection_pushdown = false;
 		table_function.filter_pushdown = false;
 		table_function.filter_prune = false;