duckdb 0.7.2-dev3353.0 → 0.7.2-dev3402.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/duckdb/extension/json/buffered_json_reader.cpp +2 -3
- package/src/duckdb/extension/json/include/json_functions.hpp +5 -1
- package/src/duckdb/extension/json/include/json_scan.hpp +1 -0
- package/src/duckdb/extension/json/include/json_transform.hpp +2 -2
- package/src/duckdb/extension/json/json-extension.cpp +7 -3
- package/src/duckdb/extension/json/json_functions/copy_json.cpp +16 -5
- package/src/duckdb/extension/json/json_functions/json_create.cpp +220 -93
- package/src/duckdb/extension/json/json_functions/json_merge_patch.cpp +2 -2
- package/src/duckdb/extension/json/json_functions/json_transform.cpp +283 -117
- package/src/duckdb/extension/json/json_functions/read_json.cpp +8 -6
- package/src/duckdb/extension/json/json_functions.cpp +17 -15
- package/src/duckdb/extension/json/json_scan.cpp +8 -4
- package/src/duckdb/extension/parquet/column_reader.cpp +6 -2
- package/src/duckdb/extension/parquet/include/parquet_reader.hpp +1 -2
- package/src/duckdb/extension/parquet/include/parquet_writer.hpp +2 -2
- package/src/duckdb/extension/parquet/include/string_column_reader.hpp +1 -0
- package/src/duckdb/extension/parquet/include/thrift_tools.hpp +3 -5
- package/src/duckdb/extension/parquet/parquet-extension.cpp +2 -4
- package/src/duckdb/extension/parquet/parquet_reader.cpp +11 -22
- package/src/duckdb/extension/parquet/parquet_statistics.cpp +5 -0
- package/src/duckdb/extension/parquet/parquet_writer.cpp +4 -4
- package/src/duckdb/src/common/file_system.cpp +13 -20
- package/src/duckdb/src/common/serializer/buffered_file_writer.cpp +2 -2
- package/src/duckdb/src/execution/index/art/art.cpp +3 -1
- package/src/duckdb/src/execution/operator/join/physical_index_join.cpp +0 -1
- package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp +2 -2
- package/src/duckdb/src/execution/operator/persistent/parallel_csv_reader.cpp +1 -1
- package/src/duckdb/src/execution/operator/persistent/physical_copy_to_file.cpp +1 -2
- package/src/duckdb/src/execution/operator/persistent/physical_export.cpp +4 -5
- package/src/duckdb/src/execution/physical_plan/plan_copy_to_file.cpp +1 -1
- package/src/duckdb/src/function/cast/cast_function_set.cpp +89 -25
- package/src/duckdb/src/function/pragma/pragma_queries.cpp +20 -15
- package/src/duckdb/src/function/table/copy_csv.cpp +4 -5
- package/src/duckdb/src/function/table/read_csv.cpp +6 -5
- package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
- package/src/duckdb/src/include/duckdb/common/file_opener.hpp +0 -1
- package/src/duckdb/src/include/duckdb/common/file_system.hpp +7 -6
- package/src/duckdb/src/include/duckdb/common/opener_file_system.hpp +118 -0
- package/src/duckdb/src/include/duckdb/common/serializer/buffered_file_writer.hpp +1 -2
- package/src/duckdb/src/include/duckdb/common/types/type_map.hpp +19 -1
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/base_csv_reader.hpp +3 -2
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/csv_line_info.hpp +1 -0
- package/src/duckdb/src/include/duckdb/main/client_data.hpp +4 -0
- package/src/duckdb/src/include/duckdb/main/extension_helper.hpp +5 -5
- package/src/duckdb/src/main/client_context.cpp +0 -4
- package/src/duckdb/src/main/client_data.cpp +19 -0
- package/src/duckdb/src/main/database.cpp +4 -1
- package/src/duckdb/src/main/extension/extension_install.cpp +5 -6
- package/src/duckdb/src/main/extension/extension_load.cpp +11 -16
- package/src/duckdb/src/main/settings/settings.cpp +2 -3
@@ -1,8 +1,9 @@
|
|
1
1
|
#include "duckdb/function/cast/cast_function_set.hpp"
|
2
|
-
|
3
|
-
#include "duckdb/common/types/type_map.hpp"
|
2
|
+
|
4
3
|
#include "duckdb/common/pair.hpp"
|
4
|
+
#include "duckdb/common/types/type_map.hpp"
|
5
5
|
#include "duckdb/function/cast_rules.hpp"
|
6
|
+
#include "duckdb/main/config.hpp"
|
6
7
|
|
7
8
|
namespace duckdb {
|
8
9
|
|
@@ -65,19 +66,90 @@ struct MapCastNode {
|
|
65
66
|
int64_t implicit_cast_cost;
|
66
67
|
};
|
67
68
|
|
69
|
+
template <class MAP_VALUE_TYPE>
|
70
|
+
static auto RelaxedTypeMatch(type_map_t<MAP_VALUE_TYPE> &map, const LogicalType &type) -> decltype(map.find(type)) {
|
71
|
+
D_ASSERT(map.find(type) == map.end()); // we shouldn't be here
|
72
|
+
switch (type.id()) {
|
73
|
+
case LogicalTypeId::LIST:
|
74
|
+
return map.find(LogicalType::LIST(LogicalType::ANY));
|
75
|
+
case LogicalTypeId::STRUCT:
|
76
|
+
return map.find(LogicalType::STRUCT({{"any", LogicalType::ANY}}));
|
77
|
+
case LogicalTypeId::MAP:
|
78
|
+
for (auto it = map.begin(); it != map.end(); it++) {
|
79
|
+
const auto &entry_type = it->first;
|
80
|
+
if (entry_type.id() != LogicalTypeId::MAP) {
|
81
|
+
continue;
|
82
|
+
}
|
83
|
+
auto &entry_key_type = MapType::KeyType(entry_type);
|
84
|
+
auto &entry_val_type = MapType::ValueType(entry_type);
|
85
|
+
if ((entry_key_type == LogicalType::ANY || entry_key_type == MapType::KeyType(type)) &&
|
86
|
+
(entry_val_type == LogicalType::ANY || entry_val_type == MapType::ValueType(type))) {
|
87
|
+
return it;
|
88
|
+
}
|
89
|
+
}
|
90
|
+
return map.end();
|
91
|
+
case LogicalTypeId::UNION:
|
92
|
+
return map.find(LogicalType::UNION({{"any", LogicalType::ANY}}));
|
93
|
+
default:
|
94
|
+
return map.find(LogicalType::ANY);
|
95
|
+
}
|
96
|
+
}
|
97
|
+
|
68
98
|
struct MapCastInfo : public BindCastInfo {
|
69
|
-
|
99
|
+
public:
|
100
|
+
const optional_ptr<MapCastNode> GetEntry(const LogicalType &source, const LogicalType &target) {
|
101
|
+
auto source_type_id_entry = casts.find(source.id());
|
102
|
+
if (source_type_id_entry == casts.end()) {
|
103
|
+
source_type_id_entry = casts.find(LogicalTypeId::ANY);
|
104
|
+
if (source_type_id_entry == casts.end()) {
|
105
|
+
return nullptr;
|
106
|
+
}
|
107
|
+
}
|
108
|
+
|
109
|
+
auto &source_type_entries = source_type_id_entry->second;
|
110
|
+
auto source_type_entry = source_type_entries.find(source);
|
111
|
+
if (source_type_entry == source_type_entries.end()) {
|
112
|
+
source_type_entry = RelaxedTypeMatch(source_type_entries, source);
|
113
|
+
if (source_type_entry == source_type_entries.end()) {
|
114
|
+
return nullptr;
|
115
|
+
}
|
116
|
+
}
|
117
|
+
|
118
|
+
auto &target_type_id_entries = source_type_entry->second;
|
119
|
+
auto target_type_id_entry = target_type_id_entries.find(target.id());
|
120
|
+
if (target_type_id_entry == target_type_id_entries.end()) {
|
121
|
+
target_type_id_entry = target_type_id_entries.find(LogicalTypeId::ANY);
|
122
|
+
if (target_type_id_entry == target_type_id_entries.end()) {
|
123
|
+
return nullptr;
|
124
|
+
}
|
125
|
+
}
|
126
|
+
|
127
|
+
auto &target_type_entries = target_type_id_entry->second;
|
128
|
+
auto target_type_entry = target_type_entries.find(target);
|
129
|
+
if (target_type_entry == target_type_entries.end()) {
|
130
|
+
target_type_entry = RelaxedTypeMatch(target_type_entries, target);
|
131
|
+
if (target_type_entry == target_type_entries.end()) {
|
132
|
+
return nullptr;
|
133
|
+
}
|
134
|
+
}
|
135
|
+
|
136
|
+
return &target_type_entry->second;
|
137
|
+
}
|
138
|
+
|
139
|
+
void AddEntry(const LogicalType &source, const LogicalType &target, MapCastNode node) {
|
140
|
+
casts[source.id()][source][target.id()].insert(make_pair(target, std::move(node)));
|
141
|
+
}
|
142
|
+
|
143
|
+
private:
|
144
|
+
type_id_map_t<type_map_t<type_id_map_t<type_map_t<MapCastNode>>>> casts;
|
70
145
|
};
|
71
146
|
|
72
147
|
int64_t CastFunctionSet::ImplicitCastCost(const LogicalType &source, const LogicalType &target) {
|
73
148
|
// check if a cast has been registered
|
74
149
|
if (map_info) {
|
75
|
-
auto
|
76
|
-
if (
|
77
|
-
|
78
|
-
if (target_entry != source_entry->second.end()) {
|
79
|
-
return target_entry->second.implicit_cast_cost;
|
80
|
-
}
|
150
|
+
auto entry = map_info->GetEntry(source, target);
|
151
|
+
if (entry) {
|
152
|
+
return entry->implicit_cast_cost;
|
81
153
|
}
|
82
154
|
}
|
83
155
|
// if not, fallback to the default implicit cast rules
|
@@ -87,22 +159,14 @@ int64_t CastFunctionSet::ImplicitCastCost(const LogicalType &source, const Logic
|
|
87
159
|
BoundCastInfo MapCastFunction(BindCastInput &input, const LogicalType &source, const LogicalType &target) {
|
88
160
|
D_ASSERT(input.info);
|
89
161
|
auto &map_info = (MapCastInfo &)*input.info;
|
90
|
-
auto
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
return
|
96
|
-
}
|
97
|
-
auto target_entry = entry->second.find(target);
|
98
|
-
if (target_entry == entry->second.end()) {
|
99
|
-
// target type not found
|
100
|
-
return nullptr;
|
101
|
-
}
|
102
|
-
if (target_entry->second.bind_function) {
|
103
|
-
return target_entry->second.bind_function(input, source, target);
|
162
|
+
auto entry = map_info.GetEntry(source, target);
|
163
|
+
if (entry) {
|
164
|
+
if (entry->bind_function) {
|
165
|
+
return entry->bind_function(input, source, target);
|
166
|
+
}
|
167
|
+
return entry->cast_info.Copy();
|
104
168
|
}
|
105
|
-
return
|
169
|
+
return nullptr;
|
106
170
|
}
|
107
171
|
|
108
172
|
void CastFunctionSet::RegisterCastFunction(const LogicalType &source, const LogicalType &target, BoundCastInfo function,
|
@@ -122,7 +186,7 @@ void CastFunctionSet::RegisterCastFunction(const LogicalType &source, const Logi
|
|
122
186
|
map_info = info.get();
|
123
187
|
bind_functions.emplace_back(MapCastFunction, std::move(info));
|
124
188
|
}
|
125
|
-
map_info->
|
189
|
+
map_info->AddEntry(source, target, std::move(node));
|
126
190
|
}
|
127
191
|
|
128
192
|
} // namespace duckdb
|
@@ -22,20 +22,26 @@ string PragmaShowTables(ClientContext &context, const FunctionParameters ¶me
|
|
22
22
|
auto schema = ClientData::Get(context).catalog_search_path->GetDefault().schema;
|
23
23
|
schema = (schema == INVALID_SCHEMA) ? DEFAULT_SCHEMA : schema; // NOLINT
|
24
24
|
|
25
|
-
auto where_clause =
|
26
|
-
StringUtil::Join({"where database_name = '", catalog, "' and schema_name = '", schema, "'"}, "");
|
25
|
+
auto where_clause = StringUtil::Format("where ((database_name = '%s') and (schema_name = '%s'))", catalog, schema);
|
27
26
|
// clang-format off
|
28
|
-
auto pragma_query = StringUtil::
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
27
|
+
auto pragma_query = StringUtil::Format(R"EOF(
|
28
|
+
with "tables" as
|
29
|
+
(
|
30
|
+
SELECT table_name as "name"
|
31
|
+
FROM duckdb_tables %s
|
32
|
+
), "views" as
|
33
|
+
(
|
34
|
+
SELECT view_name as "name"
|
35
|
+
FROM duckdb_views %s
|
36
|
+
), db_objects as
|
37
|
+
(
|
38
|
+
SELECT "name" FROM "tables"
|
39
|
+
UNION ALL
|
40
|
+
SELECT "name" FROM "views"
|
41
|
+
)
|
42
|
+
SELECT "name"
|
43
|
+
FROM db_objects
|
44
|
+
ORDER BY "name";)EOF", where_clause, where_clause, where_clause);
|
39
45
|
// clang-format on
|
40
46
|
|
41
47
|
return pragma_query;
|
@@ -124,7 +130,6 @@ string PragmaImportDatabase(ClientContext &context, const FunctionParameters &pa
|
|
124
130
|
throw PermissionException("Import is disabled through configuration");
|
125
131
|
}
|
126
132
|
auto &fs = FileSystem::GetFileSystem(context);
|
127
|
-
auto *opener = FileSystem::GetFileOpener(context);
|
128
133
|
|
129
134
|
string final_query;
|
130
135
|
// read the "shema.sql" and "load.sql" files
|
@@ -132,7 +137,7 @@ string PragmaImportDatabase(ClientContext &context, const FunctionParameters &pa
|
|
132
137
|
for (auto &file : files) {
|
133
138
|
auto file_path = fs.JoinPath(parameters.values[0].ToString(), file);
|
134
139
|
auto handle = fs.OpenFile(file_path, FileFlags::FILE_FLAGS_READ, FileSystem::DEFAULT_LOCK,
|
135
|
-
FileSystem::DEFAULT_COMPRESSION
|
140
|
+
FileSystem::DEFAULT_COMPRESSION);
|
136
141
|
auto fsize = fs.GetFileSize(*handle);
|
137
142
|
auto buffer = unique_ptr<char[]>(new char[fsize]);
|
138
143
|
fs.Read(*handle, buffer.get(), fsize);
|
@@ -254,10 +254,9 @@ struct LocalWriteCSVData : public LocalFunctionData {
|
|
254
254
|
};
|
255
255
|
|
256
256
|
struct GlobalWriteCSVData : public GlobalFunctionData {
|
257
|
-
GlobalWriteCSVData(FileSystem &fs, const string &file_path,
|
258
|
-
: fs(fs) {
|
257
|
+
GlobalWriteCSVData(FileSystem &fs, const string &file_path, FileCompressionType compression) : fs(fs) {
|
259
258
|
handle = fs.OpenFile(file_path, FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE_NEW,
|
260
|
-
FileLockType::WRITE_LOCK, compression
|
259
|
+
FileLockType::WRITE_LOCK, compression);
|
261
260
|
}
|
262
261
|
|
263
262
|
void WriteData(const_data_ptr_t data, idx_t size) {
|
@@ -288,8 +287,8 @@ static unique_ptr<GlobalFunctionData> WriteCSVInitializeGlobal(ClientContext &co
|
|
288
287
|
const string &file_path) {
|
289
288
|
auto &csv_data = bind_data.Cast<WriteCSVData>();
|
290
289
|
auto &options = csv_data.options;
|
291
|
-
auto global_data =
|
292
|
-
|
290
|
+
auto global_data =
|
291
|
+
make_uniq<GlobalWriteCSVData>(FileSystem::GetFileSystem(context), file_path, options.compression);
|
293
292
|
|
294
293
|
if (options.header) {
|
295
294
|
BufferedSerializer serializer;
|
@@ -23,9 +23,7 @@ namespace duckdb {
|
|
23
23
|
unique_ptr<CSVFileHandle> ReadCSV::OpenCSV(const string &file_path, FileCompressionType compression,
|
24
24
|
ClientContext &context) {
|
25
25
|
auto &fs = FileSystem::GetFileSystem(context);
|
26
|
-
auto
|
27
|
-
auto file_handle =
|
28
|
-
fs.OpenFile(file_path.c_str(), FileFlags::FILE_FLAGS_READ, FileLockType::NO_LOCK, compression, opener);
|
26
|
+
auto file_handle = fs.OpenFile(file_path.c_str(), FileFlags::FILE_FLAGS_READ, FileLockType::NO_LOCK, compression);
|
29
27
|
if (file_handle->CanSeek()) {
|
30
28
|
file_handle->Reset();
|
31
29
|
}
|
@@ -600,6 +598,7 @@ bool LineInfo::CanItGetLine(idx_t file_idx, idx_t batch_idx) {
|
|
600
598
|
return false;
|
601
599
|
}
|
602
600
|
|
601
|
+
// Returns the 1-indexed line number
|
603
602
|
idx_t LineInfo::GetLine(idx_t batch_idx, idx_t line_error, idx_t file_idx, idx_t cur_start, bool verify) {
|
604
603
|
unique_ptr<lock_guard<mutex>> parallel_lock;
|
605
604
|
if (!verify) {
|
@@ -607,7 +606,8 @@ idx_t LineInfo::GetLine(idx_t batch_idx, idx_t line_error, idx_t file_idx, idx_t
|
|
607
606
|
}
|
608
607
|
idx_t line_count = 0;
|
609
608
|
if (done) {
|
610
|
-
return
|
609
|
+
// line count is 0-indexed, but we want to return 1-indexed
|
610
|
+
return first_line + 1;
|
611
611
|
}
|
612
612
|
for (idx_t i = 0; i <= batch_idx; i++) {
|
613
613
|
if (lines_read.find(i) == lines_read.end() && i != batch_idx) {
|
@@ -622,7 +622,8 @@ idx_t LineInfo::GetLine(idx_t batch_idx, idx_t line_error, idx_t file_idx, idx_t
|
|
622
622
|
}
|
623
623
|
done = true;
|
624
624
|
first_line = line_count + line_error;
|
625
|
-
return
|
625
|
+
// line count is 0-indexed, but we want to return 1-indexed
|
626
|
+
return first_line + 1;
|
626
627
|
}
|
627
628
|
|
628
629
|
static unique_ptr<GlobalTableFunctionState> ParallelCSVInitGlobal(ClientContext &context,
|
@@ -1,8 +1,8 @@
|
|
1
1
|
#ifndef DUCKDB_VERSION
|
2
|
-
#define DUCKDB_VERSION "0.7.2-
|
2
|
+
#define DUCKDB_VERSION "0.7.2-dev3402"
|
3
3
|
#endif
|
4
4
|
#ifndef DUCKDB_SOURCE_ID
|
5
|
-
#define DUCKDB_SOURCE_ID "
|
5
|
+
#define DUCKDB_SOURCE_ID "6f543cb464"
|
6
6
|
#endif
|
7
7
|
#include "duckdb/function/table/system_functions.hpp"
|
8
8
|
#include "duckdb/main/database.hpp"
|
@@ -24,7 +24,6 @@ public:
|
|
24
24
|
virtual bool TryGetCurrentSetting(const string &key, Value &result) = 0;
|
25
25
|
virtual ClientContext *TryGetClientContext() = 0;
|
26
26
|
|
27
|
-
DUCKDB_API static FileOpener *Get(ClientContext &context);
|
28
27
|
DUCKDB_API static ClientContext *TryGetClientContext(FileOpener *opener);
|
29
28
|
DUCKDB_API static bool TryGetCurrentSetting(FileOpener *opener, const string &key, Value &result);
|
30
29
|
};
|
@@ -15,6 +15,7 @@
|
|
15
15
|
#include "duckdb/common/unordered_map.hpp"
|
16
16
|
#include "duckdb/common/vector.hpp"
|
17
17
|
#include "duckdb/common/enums/file_glob_options.hpp"
|
18
|
+
#include "duckdb/common/optional_ptr.hpp"
|
18
19
|
#include <functional>
|
19
20
|
|
20
21
|
#undef CreateDirectory
|
@@ -109,7 +110,6 @@ public:
|
|
109
110
|
DUCKDB_API static FileSystem &GetFileSystem(ClientContext &context);
|
110
111
|
DUCKDB_API static FileSystem &GetFileSystem(DatabaseInstance &db);
|
111
112
|
DUCKDB_API static FileSystem &Get(AttachedDatabase &db);
|
112
|
-
DUCKDB_API static FileOpener *GetFileOpener(ClientContext &context);
|
113
113
|
|
114
114
|
DUCKDB_API virtual unique_ptr<FileHandle> OpenFile(const string &path, uint8_t flags,
|
115
115
|
FileLockType lock = DEFAULT_LOCK,
|
@@ -128,8 +128,6 @@ public:
|
|
128
128
|
//! Write nr_bytes from the buffer into the file, moving the file pointer forward by nr_bytes.
|
129
129
|
DUCKDB_API virtual int64_t Write(FileHandle &handle, void *buffer, int64_t nr_bytes);
|
130
130
|
|
131
|
-
//! Returns the extension of the file, or empty string if no extension was found.
|
132
|
-
DUCKDB_API string GetFileExtension(FileHandle &handle);
|
133
131
|
//! Returns the file size of a file handle, returns -1 on error
|
134
132
|
DUCKDB_API virtual int64_t GetFileSize(FileHandle &handle);
|
135
133
|
//! Returns the file last modified time of a file handle, returns timespec with zero on all attributes on error
|
@@ -167,9 +165,13 @@ public:
|
|
167
165
|
//! Gets the working directory
|
168
166
|
DUCKDB_API static string GetWorkingDirectory();
|
169
167
|
//! Gets the users home directory
|
170
|
-
DUCKDB_API static string GetHomeDirectory(FileOpener
|
168
|
+
DUCKDB_API static string GetHomeDirectory(optional_ptr<FileOpener> opener);
|
169
|
+
//! Gets the users home directory
|
170
|
+
DUCKDB_API virtual string GetHomeDirectory();
|
171
|
+
//! Expands a given path, including e.g. expanding the home directory of the user
|
172
|
+
DUCKDB_API static string ExpandPath(const string &path, optional_ptr<FileOpener> opener);
|
171
173
|
//! Expands a given path, including e.g. expanding the home directory of the user
|
172
|
-
DUCKDB_API
|
174
|
+
DUCKDB_API virtual string ExpandPath(const string &path);
|
173
175
|
//! Returns the system-available memory in bytes. Returns DConstants::INVALID_INDEX if the system function fails.
|
174
176
|
DUCKDB_API static idx_t GetAvailableMemory();
|
175
177
|
//! Path separator for the current file system
|
@@ -187,7 +189,6 @@ public:
|
|
187
189
|
|
188
190
|
//! Runs a glob on the file system, returning a list of matching files
|
189
191
|
DUCKDB_API virtual vector<string> Glob(const string &path, FileOpener *opener = nullptr);
|
190
|
-
DUCKDB_API virtual vector<string> Glob(const string &path, ClientContext &context);
|
191
192
|
DUCKDB_API vector<string> GlobFiles(const string &path, ClientContext &context,
|
192
193
|
FileGlobOptions options = FileGlobOptions::DISALLOW_EMPTY);
|
193
194
|
|
@@ -0,0 +1,118 @@
|
|
1
|
+
//===----------------------------------------------------------------------===//
|
2
|
+
// DuckDB
|
3
|
+
//
|
4
|
+
// duckdb/common/opener_file_system.hpp
|
5
|
+
//
|
6
|
+
//
|
7
|
+
//===----------------------------------------------------------------------===//
|
8
|
+
|
9
|
+
#pragma once
|
10
|
+
|
11
|
+
#include "duckdb/common/file_system.hpp"
|
12
|
+
|
13
|
+
namespace duckdb {
|
14
|
+
|
15
|
+
// The OpenerFileSystem is wrapper for a file system that pushes an appropriate FileOpener into the various API calls
|
16
|
+
class OpenerFileSystem : public FileSystem {
|
17
|
+
public:
|
18
|
+
virtual FileSystem &GetFileSystem() const = 0;
|
19
|
+
virtual optional_ptr<FileOpener> GetOpener() const = 0;
|
20
|
+
|
21
|
+
unique_ptr<FileHandle> OpenFile(const string &path, uint8_t flags, FileLockType lock = FileLockType::NO_LOCK,
|
22
|
+
FileCompressionType compression = FileCompressionType::UNCOMPRESSED,
|
23
|
+
FileOpener *opener = nullptr) override {
|
24
|
+
if (opener) {
|
25
|
+
throw InternalException("OpenerFileSystem cannot take an opener - the opener is pushed automatically");
|
26
|
+
}
|
27
|
+
return GetFileSystem().OpenFile(path, flags, lock, compression, GetOpener().get());
|
28
|
+
}
|
29
|
+
|
30
|
+
void Read(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) override {
|
31
|
+
GetFileSystem().Read(handle, buffer, nr_bytes, location);
|
32
|
+
};
|
33
|
+
|
34
|
+
void Write(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) override {
|
35
|
+
GetFileSystem().Write(handle, buffer, nr_bytes, location);
|
36
|
+
}
|
37
|
+
|
38
|
+
int64_t Read(FileHandle &handle, void *buffer, int64_t nr_bytes) override {
|
39
|
+
return GetFileSystem().Read(handle, buffer, nr_bytes);
|
40
|
+
}
|
41
|
+
|
42
|
+
int64_t Write(FileHandle &handle, void *buffer, int64_t nr_bytes) override {
|
43
|
+
return GetFileSystem().Write(handle, buffer, nr_bytes);
|
44
|
+
}
|
45
|
+
|
46
|
+
int64_t GetFileSize(FileHandle &handle) override {
|
47
|
+
return GetFileSystem().GetFileSize(handle);
|
48
|
+
}
|
49
|
+
time_t GetLastModifiedTime(FileHandle &handle) override {
|
50
|
+
return GetFileSystem().GetLastModifiedTime(handle);
|
51
|
+
}
|
52
|
+
FileType GetFileType(FileHandle &handle) override {
|
53
|
+
return GetFileSystem().GetFileType(handle);
|
54
|
+
}
|
55
|
+
|
56
|
+
void Truncate(FileHandle &handle, int64_t new_size) override {
|
57
|
+
GetFileSystem().Truncate(handle, new_size);
|
58
|
+
}
|
59
|
+
|
60
|
+
void FileSync(FileHandle &handle) override {
|
61
|
+
GetFileSystem().FileSync(handle);
|
62
|
+
}
|
63
|
+
|
64
|
+
bool DirectoryExists(const string &directory) override {
|
65
|
+
return GetFileSystem().DirectoryExists(directory);
|
66
|
+
}
|
67
|
+
void CreateDirectory(const string &directory) override {
|
68
|
+
return GetFileSystem().CreateDirectory(directory);
|
69
|
+
}
|
70
|
+
|
71
|
+
void RemoveDirectory(const string &directory) override {
|
72
|
+
return GetFileSystem().RemoveDirectory(directory);
|
73
|
+
}
|
74
|
+
|
75
|
+
bool ListFiles(const string &directory, const std::function<void(const string &, bool)> &callback,
|
76
|
+
FileOpener *opener = nullptr) override {
|
77
|
+
if (opener) {
|
78
|
+
throw InternalException("OpenerFileSystem cannot take an opener - the opener is pushed automatically");
|
79
|
+
}
|
80
|
+
return GetFileSystem().ListFiles(directory, callback, GetOpener().get());
|
81
|
+
}
|
82
|
+
|
83
|
+
void MoveFile(const string &source, const string &target) override {
|
84
|
+
GetFileSystem().MoveFile(source, target);
|
85
|
+
}
|
86
|
+
|
87
|
+
string GetHomeDirectory() override {
|
88
|
+
return FileSystem::GetHomeDirectory(GetOpener());
|
89
|
+
}
|
90
|
+
|
91
|
+
string ExpandPath(const string &path) override {
|
92
|
+
return FileSystem::ExpandPath(path, GetOpener());
|
93
|
+
}
|
94
|
+
|
95
|
+
bool FileExists(const string &filename) override {
|
96
|
+
return GetFileSystem().FileExists(filename);
|
97
|
+
}
|
98
|
+
|
99
|
+
bool IsPipe(const string &filename) override {
|
100
|
+
return GetFileSystem().IsPipe(filename);
|
101
|
+
}
|
102
|
+
virtual void RemoveFile(const string &filename) override {
|
103
|
+
GetFileSystem().RemoveFile(filename);
|
104
|
+
}
|
105
|
+
|
106
|
+
virtual vector<string> Glob(const string &path, FileOpener *opener = nullptr) override {
|
107
|
+
if (opener) {
|
108
|
+
throw InternalException("OpenerFileSystem cannot take an opener - the opener is pushed automatically");
|
109
|
+
}
|
110
|
+
return GetFileSystem().Glob(path, GetOpener().get());
|
111
|
+
}
|
112
|
+
|
113
|
+
std::string GetName() const override {
|
114
|
+
return "OpenerFileSystem - " + GetFileSystem().GetName();
|
115
|
+
}
|
116
|
+
};
|
117
|
+
|
118
|
+
} // namespace duckdb
|
@@ -21,8 +21,7 @@ public:
|
|
21
21
|
|
22
22
|
//! Serializes to a buffer allocated by the serializer, will expand when
|
23
23
|
//! writing past the initial threshold
|
24
|
-
DUCKDB_API BufferedFileWriter(FileSystem &fs, const string &path, uint8_t open_flags = DEFAULT_OPEN_FLAGS
|
25
|
-
FileOpener *opener = nullptr);
|
24
|
+
DUCKDB_API BufferedFileWriter(FileSystem &fs, const string &path, uint8_t open_flags = DEFAULT_OPEN_FLAGS);
|
26
25
|
|
27
26
|
FileSystem &fs;
|
28
27
|
string path;
|
@@ -8,9 +8,10 @@
|
|
8
8
|
|
9
9
|
#pragma once
|
10
10
|
|
11
|
+
#include "duckdb/common/types.hpp"
|
12
|
+
#include "duckdb/common/types/hash.hpp"
|
11
13
|
#include "duckdb/common/unordered_map.hpp"
|
12
14
|
#include "duckdb/common/unordered_set.hpp"
|
13
|
-
#include "duckdb/common/types.hpp"
|
14
15
|
|
15
16
|
namespace duckdb {
|
16
17
|
|
@@ -31,4 +32,21 @@ using type_map_t = unordered_map<LogicalType, T, LogicalTypeHashFunction, Logica
|
|
31
32
|
|
32
33
|
using type_set_t = unordered_set<LogicalType, LogicalTypeHashFunction, LogicalTypeEquality>;
|
33
34
|
|
35
|
+
struct LogicalTypeIdHashFunction {
|
36
|
+
uint64_t operator()(const LogicalTypeId &type_id) const {
|
37
|
+
return duckdb::Hash<uint8_t>((uint8_t)type_id);
|
38
|
+
}
|
39
|
+
};
|
40
|
+
|
41
|
+
struct LogicalTypeIdEquality {
|
42
|
+
bool operator()(const LogicalTypeId &a, const LogicalTypeId &b) const {
|
43
|
+
return a == b;
|
44
|
+
}
|
45
|
+
};
|
46
|
+
|
47
|
+
template <typename T>
|
48
|
+
using type_id_map_t = unordered_map<LogicalTypeId, T, LogicalTypeIdHashFunction, LogicalTypeIdEquality>;
|
49
|
+
|
50
|
+
using type_id_set_t = unordered_set<LogicalTypeId, LogicalTypeIdHashFunction, LogicalTypeIdEquality>;
|
51
|
+
|
34
52
|
} // namespace duckdb
|
@@ -42,7 +42,6 @@ public:
|
|
42
42
|
ClientContext &context;
|
43
43
|
FileSystem &fs;
|
44
44
|
Allocator &allocator;
|
45
|
-
FileOpener *opener;
|
46
45
|
BufferedCSVReaderOptions options;
|
47
46
|
vector<LogicalType> return_types;
|
48
47
|
vector<string> names;
|
@@ -76,8 +75,10 @@ public:
|
|
76
75
|
const vector<LogicalType> &GetTypes() {
|
77
76
|
return return_types;
|
78
77
|
}
|
78
|
+
|
79
|
+
//! Get the 1-indexed global line number for the given local error line
|
79
80
|
virtual idx_t GetLineError(idx_t line_error, idx_t buffer_idx) {
|
80
|
-
return line_error;
|
81
|
+
return line_error + 1;
|
81
82
|
};
|
82
83
|
|
83
84
|
//! Initialize projection indices to select all columns
|
@@ -17,6 +17,7 @@ public:
|
|
17
17
|
tuple_end(tuple_end_p) {};
|
18
18
|
bool CanItGetLine(idx_t file_idx, idx_t batch_idx);
|
19
19
|
|
20
|
+
//! Return the 1-indexed line number
|
20
21
|
idx_t GetLine(idx_t batch_idx, idx_t line_error = 0, idx_t file_idx = 0, idx_t cur_start = 0, bool verify = true);
|
21
22
|
//! Verify if the CSV File was read correctly from [0,batch_idx] batches.
|
22
23
|
void Verify(idx_t file_idx, idx_t batch_idx, idx_t cur_first_pos);
|
@@ -20,6 +20,7 @@ class BufferedFileWriter;
|
|
20
20
|
class ClientContext;
|
21
21
|
class CatalogSearchPath;
|
22
22
|
class FileOpener;
|
23
|
+
class FileSystem;
|
23
24
|
class HTTPState;
|
24
25
|
class QueryProfiler;
|
25
26
|
class QueryProfilerHistory;
|
@@ -55,6 +56,9 @@ struct ClientData {
|
|
55
56
|
//! HTTP State in this query
|
56
57
|
unique_ptr<HTTPState> http_state;
|
57
58
|
|
59
|
+
//! The clients' file system wrapper
|
60
|
+
unique_ptr<FileSystem> client_file_system;
|
61
|
+
|
58
62
|
//! The file search path
|
59
63
|
string file_search_path;
|
60
64
|
|
@@ -43,10 +43,10 @@ public:
|
|
43
43
|
static void InstallExtension(ClientContext &context, const string &extension, bool force_install);
|
44
44
|
static void InstallExtension(DBConfig &config, FileSystem &fs, const string &extension, bool force_install);
|
45
45
|
static void LoadExternalExtension(ClientContext &context, const string &extension);
|
46
|
-
static void LoadExternalExtension(DatabaseInstance &db,
|
46
|
+
static void LoadExternalExtension(DatabaseInstance &db, FileSystem &fs, const string &extension);
|
47
47
|
|
48
48
|
static string ExtensionDirectory(ClientContext &context);
|
49
|
-
static string ExtensionDirectory(DBConfig &config, FileSystem &fs
|
49
|
+
static string ExtensionDirectory(DBConfig &config, FileSystem &fs);
|
50
50
|
|
51
51
|
static idx_t DefaultExtensionCount();
|
52
52
|
static DefaultExtension GetDefaultExtension(idx_t index);
|
@@ -70,9 +70,9 @@ private:
|
|
70
70
|
const string &local_path, const string &extension, bool force_install);
|
71
71
|
static const vector<string> PathComponents();
|
72
72
|
static bool AllowAutoInstall(const string &extension);
|
73
|
-
static ExtensionInitResult InitialLoad(DBConfig &config,
|
74
|
-
static bool TryInitialLoad(DBConfig &config,
|
75
|
-
|
73
|
+
static ExtensionInitResult InitialLoad(DBConfig &config, FileSystem &fs, const string &extension);
|
74
|
+
static bool TryInitialLoad(DBConfig &config, FileSystem &fs, const string &extension, ExtensionInitResult &result,
|
75
|
+
string &error);
|
76
76
|
//! For tagged releases we use the tag, else we use the git commit hash
|
77
77
|
static const string GetVersionDirectoryName();
|
78
78
|
//! Version tags occur with and without 'v', tag in extension path is always with 'v'
|
@@ -233,10 +233,6 @@ Executor &ClientContext::GetExecutor() {
|
|
233
233
|
return *active_query->executor;
|
234
234
|
}
|
235
235
|
|
236
|
-
FileOpener *FileOpener::Get(ClientContext &context) {
|
237
|
-
return ClientData::Get(context).file_opener.get();
|
238
|
-
}
|
239
|
-
|
240
236
|
const string &ClientContext::GetCurrentQuery() {
|
241
237
|
D_ASSERT(active_query);
|
242
238
|
return active_query->query;
|
@@ -11,9 +11,27 @@
|
|
11
11
|
#include "duckdb/main/database.hpp"
|
12
12
|
#include "duckdb/main/database_manager.hpp"
|
13
13
|
#include "duckdb/main/query_profiler.hpp"
|
14
|
+
#include "duckdb/common/opener_file_system.hpp"
|
14
15
|
|
15
16
|
namespace duckdb {
|
16
17
|
|
18
|
+
class ClientFileSystem : public OpenerFileSystem {
|
19
|
+
public:
|
20
|
+
explicit ClientFileSystem(ClientContext &context_p) : context(context_p) {
|
21
|
+
}
|
22
|
+
|
23
|
+
FileSystem &GetFileSystem() const override {
|
24
|
+
auto &config = DBConfig::GetConfig(context);
|
25
|
+
return *config.file_system;
|
26
|
+
}
|
27
|
+
optional_ptr<FileOpener> GetOpener() const override {
|
28
|
+
return ClientData::Get(context).file_opener.get();
|
29
|
+
}
|
30
|
+
|
31
|
+
private:
|
32
|
+
ClientContext &context;
|
33
|
+
};
|
34
|
+
|
17
35
|
ClientData::ClientData(ClientContext &context) : catalog_search_path(make_uniq<CatalogSearchPath>(context)) {
|
18
36
|
auto &db = DatabaseInstance::GetDatabase(context);
|
19
37
|
profiler = make_shared<QueryProfiler>(context);
|
@@ -22,6 +40,7 @@ ClientData::ClientData(ClientContext &context) : catalog_search_path(make_uniq<C
|
|
22
40
|
temporary_objects->oid = DatabaseManager::Get(db).ModifyCatalog();
|
23
41
|
random_engine = make_uniq<RandomEngine>();
|
24
42
|
file_opener = make_uniq<ClientContextFileOpener>(context);
|
43
|
+
client_file_system = make_uniq<ClientFileSystem>(context);
|
25
44
|
temporary_objects->Initialize();
|
26
45
|
}
|
27
46
|
ClientData::~ClientData() {
|
@@ -230,7 +230,10 @@ void DatabaseInstance::Initialize(const char *database_path, DBConfig *user_conf
|
|
230
230
|
|
231
231
|
if (!config.options.database_type.empty()) {
|
232
232
|
// if we are opening an extension database - load the extension
|
233
|
-
|
233
|
+
if (!config.file_system) {
|
234
|
+
throw InternalException("No file system!?");
|
235
|
+
}
|
236
|
+
ExtensionHelper::LoadExternalExtension(*this, *config.file_system, config.options.database_type);
|
234
237
|
}
|
235
238
|
|
236
239
|
if (!config.options.unrecognized_options.empty()) {
|