duckdb 0.8.2-dev1212.0 → 0.8.2-dev1435.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/duckdb/extension/parquet/parquet_extension.cpp +1 -1
- package/src/duckdb/src/common/file_system.cpp +4 -0
- package/src/duckdb/src/common/hive_partitioning.cpp +10 -6
- package/src/duckdb/src/common/multi_file_reader.cpp +3 -2
- package/src/duckdb/src/common/operator/cast_operators.cpp +35 -1
- package/src/duckdb/src/common/types/bit.cpp +51 -0
- package/src/duckdb/src/common/virtual_file_system.cpp +138 -1
- package/src/duckdb/src/core_functions/function_list.cpp +2 -2
- package/src/duckdb/src/core_functions/scalar/date/date_part.cpp +2 -2
- package/src/duckdb/src/core_functions/scalar/date/epoch.cpp +10 -7
- package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp +7 -2
- package/src/duckdb/src/execution/physical_plan/plan_get.cpp +2 -2
- package/src/duckdb/src/function/cast/bit_cast.cpp +34 -2
- package/src/duckdb/src/function/cast/blob_cast.cpp +3 -0
- package/src/duckdb/src/function/cast/numeric_casts.cpp +2 -0
- package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
- package/src/duckdb/src/include/duckdb/common/extra_operator_info.hpp +27 -0
- package/src/duckdb/src/include/duckdb/common/file_system.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/hive_partitioning.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/operator/cast_operators.hpp +43 -3
- package/src/duckdb/src/include/duckdb/common/operator/numeric_cast.hpp +10 -0
- package/src/duckdb/src/include/duckdb/common/types/bit.hpp +81 -0
- package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp +38 -97
- package/src/duckdb/src/include/duckdb/core_functions/scalar/date_functions.hpp +11 -11
- package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp +5 -1
- package/src/duckdb/src/include/duckdb/main/relation/cross_product_relation.hpp +4 -1
- package/src/duckdb/src/include/duckdb/main/relation/join_relation.hpp +5 -2
- package/src/duckdb/src/include/duckdb/main/relation.hpp +4 -2
- package/src/duckdb/src/include/duckdb/main/settings.hpp +9 -0
- package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp +4 -0
- package/src/duckdb/src/main/config.cpp +1 -0
- package/src/duckdb/src/main/extension/extension_install.cpp +6 -0
- package/src/duckdb/src/main/extension/extension_load.cpp +10 -1
- package/src/duckdb/src/main/relation/cross_product_relation.cpp +4 -3
- package/src/duckdb/src/main/relation/join_relation.cpp +5 -5
- package/src/duckdb/src/main/relation.cpp +6 -5
- package/src/duckdb/src/main/settings/settings.cpp +24 -0
- package/src/duckdb/src/parser/parser.cpp +8 -2
- package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +5 -4
- package/src/duckdb/src/planner/operator/logical_get.cpp +9 -4
package/package.json
CHANGED
@@ -175,7 +175,6 @@ public:
|
|
175
175
|
table_function.serialize = ParquetScanSerialize;
|
176
176
|
table_function.deserialize = ParquetScanDeserialize;
|
177
177
|
table_function.get_batch_info = ParquetGetBatchInfo;
|
178
|
-
|
179
178
|
table_function.projection_pushdown = true;
|
180
179
|
table_function.filter_pushdown = true;
|
181
180
|
table_function.filter_prune = true;
|
@@ -520,6 +519,7 @@ public:
|
|
520
519
|
static void ParquetComplexFilterPushdown(ClientContext &context, LogicalGet &get, FunctionData *bind_data_p,
|
521
520
|
vector<unique_ptr<Expression>> &filters) {
|
522
521
|
auto &data = bind_data_p->Cast<ParquetReadBindData>();
|
522
|
+
|
523
523
|
auto reset_reader = MultiFileReader::ComplexFilterPushdown(context, data.files,
|
524
524
|
data.parquet_options.file_options, get, filters);
|
525
525
|
if (reset_reader) {
|
@@ -385,6 +385,10 @@ void FileSystem::UnregisterSubSystem(const string &name) {
|
|
385
385
|
throw NotImplementedException("%s: Can't unregister a sub system on a non-virtual file system", GetName());
|
386
386
|
}
|
387
387
|
|
388
|
+
void FileSystem::SetDisabledFileSystems(const vector<string> &names) {
|
389
|
+
throw NotImplementedException("%s: Can't disable file systems on a non-virtual file system", GetName());
|
390
|
+
}
|
391
|
+
|
388
392
|
vector<string> FileSystem::ListSubSystems() {
|
389
393
|
throw NotImplementedException("%s: Can't list sub systems on a non-virtual file system", GetName());
|
390
394
|
}
|
@@ -6,6 +6,7 @@
|
|
6
6
|
#include "duckdb/planner/expression/bound_constant_expression.hpp"
|
7
7
|
#include "duckdb/planner/expression/bound_reference_expression.hpp"
|
8
8
|
#include "duckdb/planner/expression_iterator.hpp"
|
9
|
+
#include "duckdb/planner/operator/logical_get.hpp"
|
9
10
|
#include "duckdb/planner/table_filter.hpp"
|
10
11
|
#include "re2/re2.h"
|
11
12
|
|
@@ -86,12 +87,15 @@ std::map<string, string> HivePartitioning::Parse(const string &filename) {
|
|
86
87
|
// currently, only expressions that cannot be evaluated during pushdown are removed.
|
87
88
|
void HivePartitioning::ApplyFiltersToFileList(ClientContext &context, vector<string> &files,
|
88
89
|
vector<unique_ptr<Expression>> &filters,
|
89
|
-
unordered_map<string, column_t> &column_map,
|
90
|
+
unordered_map<string, column_t> &column_map, LogicalGet &get,
|
90
91
|
bool hive_enabled, bool filename_enabled) {
|
92
|
+
|
91
93
|
vector<string> pruned_files;
|
92
94
|
vector<bool> have_preserved_filter(filters.size(), false);
|
93
95
|
vector<unique_ptr<Expression>> pruned_filters;
|
96
|
+
unordered_set<idx_t> filters_applied_to_files;
|
94
97
|
duckdb_re2::RE2 regex(REGEX_STRING);
|
98
|
+
auto table_index = get.table_index;
|
95
99
|
|
96
100
|
if ((!filename_enabled && !hive_enabled) || filters.empty()) {
|
97
101
|
return;
|
@@ -121,11 +125,11 @@ void HivePartitioning::ApplyFiltersToFileList(ClientContext &context, vector<str
|
|
121
125
|
} else if (!result_value.GetValue<bool>()) {
|
122
126
|
// filter evaluates to false
|
123
127
|
should_prune_file = true;
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
128
|
+
// convert the filter to a table filter.
|
129
|
+
if (filters_applied_to_files.find(j) == filters_applied_to_files.end()) {
|
130
|
+
get.extra_info.file_filters += filter->ToString();
|
131
|
+
filters_applied_to_files.insert(j);
|
132
|
+
}
|
129
133
|
}
|
130
134
|
}
|
131
135
|
|
@@ -106,8 +106,9 @@ bool MultiFileReader::ComplexFilterPushdown(ClientContext &context, vector<strin
|
|
106
106
|
}
|
107
107
|
|
108
108
|
auto start_files = files.size();
|
109
|
-
HivePartitioning::ApplyFiltersToFileList(context, files, filters, column_map, get.
|
110
|
-
options.
|
109
|
+
HivePartitioning::ApplyFiltersToFileList(context, files, filters, column_map, get, options.hive_partitioning,
|
110
|
+
options.filename);
|
111
|
+
|
111
112
|
if (files.size() != start_files) {
|
112
113
|
// we have pruned files
|
113
114
|
return true;
|
@@ -1,4 +1,5 @@
|
|
1
1
|
#include "duckdb/common/operator/cast_operators.hpp"
|
2
|
+
#include "duckdb/common/hugeint.hpp"
|
2
3
|
#include "duckdb/common/operator/string_cast.hpp"
|
3
4
|
#include "duckdb/common/operator/numeric_cast.hpp"
|
4
5
|
#include "duckdb/common/operator/decimal_cast_operators.hpp"
|
@@ -1425,11 +1426,20 @@ string_t CastFromBlob::Operation(string_t input, Vector &vector) {
|
|
1425
1426
|
return result;
|
1426
1427
|
}
|
1427
1428
|
|
1429
|
+
template <>
|
1430
|
+
string_t CastFromBlobToBit::Operation(string_t input, Vector &vector) {
|
1431
|
+
idx_t result_size = input.GetSize() + 1;
|
1432
|
+
if (result_size <= 1) {
|
1433
|
+
throw ConversionException("Cannot cast empty BLOB to BIT");
|
1434
|
+
}
|
1435
|
+
return StringVector::AddStringOrBlob(vector, Bit::BlobToBit(input));
|
1436
|
+
}
|
1437
|
+
|
1428
1438
|
//===--------------------------------------------------------------------===//
|
1429
1439
|
// Cast From Bit
|
1430
1440
|
//===--------------------------------------------------------------------===//
|
1431
1441
|
template <>
|
1432
|
-
string_t
|
1442
|
+
string_t CastFromBitToString::Operation(string_t input, Vector &vector) {
|
1433
1443
|
|
1434
1444
|
idx_t result_size = Bit::BitLength(input);
|
1435
1445
|
string_t result = StringVector::EmptyString(vector, result_size);
|
@@ -1482,6 +1492,30 @@ bool TryCastToBit::Operation(string_t input, string_t &result, Vector &result_ve
|
|
1482
1492
|
return true;
|
1483
1493
|
}
|
1484
1494
|
|
1495
|
+
template <>
|
1496
|
+
bool CastFromBitToNumeric::Operation(string_t input, bool &result, bool strict) {
|
1497
|
+
D_ASSERT(input.GetSize() > 1);
|
1498
|
+
|
1499
|
+
uint8_t value;
|
1500
|
+
bool success = CastFromBitToNumeric::Operation(input, value, strict);
|
1501
|
+
result = (value > 0);
|
1502
|
+
return (success);
|
1503
|
+
}
|
1504
|
+
|
1505
|
+
template <>
|
1506
|
+
bool CastFromBitToNumeric::Operation(string_t input, hugeint_t &result, bool strict) {
|
1507
|
+
D_ASSERT(input.GetSize() > 1);
|
1508
|
+
|
1509
|
+
if (input.GetSize() - 1 > sizeof(hugeint_t)) {
|
1510
|
+
throw ConversionException("Bitstring doesn't fit inside of %s", GetTypeId<hugeint_t>());
|
1511
|
+
}
|
1512
|
+
Bit::BitToNumeric(input, result);
|
1513
|
+
if (result < NumericLimits<hugeint_t>::Minimum()) {
|
1514
|
+
throw ConversionException("Minimum limit for HUGEINT is %s", NumericLimits<hugeint_t>::Minimum().ToString());
|
1515
|
+
}
|
1516
|
+
return (true);
|
1517
|
+
}
|
1518
|
+
|
1485
1519
|
//===--------------------------------------------------------------------===//
|
1486
1520
|
// Cast From UUID
|
1487
1521
|
//===--------------------------------------------------------------------===//
|
@@ -1,4 +1,6 @@
|
|
1
|
+
#include "duckdb/common/assert.hpp"
|
1
2
|
#include "duckdb/common/operator/cast_operators.hpp"
|
3
|
+
#include "duckdb/common/typedefs.hpp"
|
2
4
|
#include "duckdb/common/types/bit.hpp"
|
3
5
|
#include "duckdb/common/types/string_type.hpp"
|
4
6
|
|
@@ -34,6 +36,13 @@ static inline idx_t GetBitSize(const string_t &str) {
|
|
34
36
|
return str_len;
|
35
37
|
}
|
36
38
|
|
39
|
+
uint8_t Bit::GetFirstByte(const string_t &str) {
|
40
|
+
D_ASSERT(str.GetSize() > 1);
|
41
|
+
|
42
|
+
auto data = const_data_ptr_cast(str.GetData());
|
43
|
+
return data[1] & ((1 << (8 - data[0])) - 1);
|
44
|
+
}
|
45
|
+
|
37
46
|
void Bit::Finalize(string_t &str) {
|
38
47
|
// bit strings require all padding bits to be set to 1
|
39
48
|
// this method sets all padding bits to 1
|
@@ -146,6 +155,48 @@ string Bit::ToBit(string_t str) {
|
|
146
155
|
return output_str.GetString();
|
147
156
|
}
|
148
157
|
|
158
|
+
void Bit::BlobToBit(string_t blob, string_t &output_str) {
|
159
|
+
auto data = const_data_ptr_cast(blob.GetData());
|
160
|
+
auto output = output_str.GetDataWriteable();
|
161
|
+
idx_t size = blob.GetSize();
|
162
|
+
|
163
|
+
*output = 0; // No padding
|
164
|
+
memcpy(output + 1, data, size);
|
165
|
+
}
|
166
|
+
|
167
|
+
string Bit::BlobToBit(string_t blob) {
|
168
|
+
auto buffer = make_unsafe_uniq_array<char>(blob.GetSize() + 1);
|
169
|
+
string_t output_str(buffer.get(), blob.GetSize() + 1);
|
170
|
+
Bit::BlobToBit(blob, output_str);
|
171
|
+
return output_str.GetString();
|
172
|
+
}
|
173
|
+
|
174
|
+
void Bit::BitToBlob(string_t bit, string_t &output_blob) {
|
175
|
+
D_ASSERT(bit.GetSize() == output_blob.GetSize() + 1);
|
176
|
+
|
177
|
+
auto data = const_data_ptr_cast(bit.GetData());
|
178
|
+
auto output = output_blob.GetDataWriteable();
|
179
|
+
idx_t size = output_blob.GetSize();
|
180
|
+
|
181
|
+
output[0] = GetFirstByte(bit);
|
182
|
+
if (size > 2) {
|
183
|
+
++output;
|
184
|
+
// First byte in bitstring contains amount of padded bits,
|
185
|
+
// second byte in bitstring is the padded byte,
|
186
|
+
// therefore the rest of the data starts at data + 2 (third byte)
|
187
|
+
memcpy(output, data + 2, size - 1);
|
188
|
+
}
|
189
|
+
}
|
190
|
+
|
191
|
+
string Bit::BitToBlob(string_t bit) {
|
192
|
+
D_ASSERT(bit.GetSize() > 1);
|
193
|
+
|
194
|
+
auto buffer = make_unsafe_uniq_array<char>(bit.GetSize() - 1);
|
195
|
+
string_t output_str(buffer.get(), bit.GetSize() - 1);
|
196
|
+
Bit::BitToBlob(bit, output_str);
|
197
|
+
return output_str.GetString();
|
198
|
+
}
|
199
|
+
|
149
200
|
// **** scalar functions ****
|
150
201
|
void Bit::BitString(const string_t &input, const idx_t &bit_length, string_t &result) {
|
151
202
|
char *res_buf = result.GetDataWriteable();
|
@@ -28,7 +28,7 @@ unique_ptr<FileHandle> VirtualFileSystem::OpenFile(const string &path, uint8_t f
|
|
28
28
|
}
|
29
29
|
}
|
30
30
|
// open the base file handle
|
31
|
-
auto file_handle = FindFileSystem(path)
|
31
|
+
auto file_handle = FindFileSystem(path).OpenFile(path, flags, lock, FileCompressionType::UNCOMPRESSED, opener);
|
32
32
|
if (file_handle->GetType() == FileType::FILE_TYPE_FIFO) {
|
33
33
|
file_handle = PipeFileSystem::OpenPipe(std::move(file_handle));
|
34
34
|
} else if (compression != FileCompressionType::UNCOMPRESSED) {
|
@@ -42,4 +42,141 @@ unique_ptr<FileHandle> VirtualFileSystem::OpenFile(const string &path, uint8_t f
|
|
42
42
|
return file_handle;
|
43
43
|
}
|
44
44
|
|
45
|
+
void VirtualFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) {
|
46
|
+
handle.file_system.Read(handle, buffer, nr_bytes, location);
|
47
|
+
}
|
48
|
+
|
49
|
+
void VirtualFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) {
|
50
|
+
handle.file_system.Write(handle, buffer, nr_bytes, location);
|
51
|
+
}
|
52
|
+
|
53
|
+
int64_t VirtualFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes) {
|
54
|
+
return handle.file_system.Read(handle, buffer, nr_bytes);
|
55
|
+
}
|
56
|
+
|
57
|
+
int64_t VirtualFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_bytes) {
|
58
|
+
return handle.file_system.Write(handle, buffer, nr_bytes);
|
59
|
+
}
|
60
|
+
|
61
|
+
int64_t VirtualFileSystem::GetFileSize(FileHandle &handle) {
|
62
|
+
return handle.file_system.GetFileSize(handle);
|
63
|
+
}
|
64
|
+
time_t VirtualFileSystem::GetLastModifiedTime(FileHandle &handle) {
|
65
|
+
return handle.file_system.GetLastModifiedTime(handle);
|
66
|
+
}
|
67
|
+
FileType VirtualFileSystem::GetFileType(FileHandle &handle) {
|
68
|
+
return handle.file_system.GetFileType(handle);
|
69
|
+
}
|
70
|
+
|
71
|
+
void VirtualFileSystem::Truncate(FileHandle &handle, int64_t new_size) {
|
72
|
+
handle.file_system.Truncate(handle, new_size);
|
73
|
+
}
|
74
|
+
|
75
|
+
void VirtualFileSystem::FileSync(FileHandle &handle) {
|
76
|
+
handle.file_system.FileSync(handle);
|
77
|
+
}
|
78
|
+
|
79
|
+
// need to look up correct fs for this
|
80
|
+
bool VirtualFileSystem::DirectoryExists(const string &directory) {
|
81
|
+
return FindFileSystem(directory).DirectoryExists(directory);
|
82
|
+
}
|
83
|
+
void VirtualFileSystem::CreateDirectory(const string &directory) {
|
84
|
+
FindFileSystem(directory).CreateDirectory(directory);
|
85
|
+
}
|
86
|
+
|
87
|
+
void VirtualFileSystem::RemoveDirectory(const string &directory) {
|
88
|
+
FindFileSystem(directory).RemoveDirectory(directory);
|
89
|
+
}
|
90
|
+
|
91
|
+
bool VirtualFileSystem::ListFiles(const string &directory, const std::function<void(const string &, bool)> &callback,
|
92
|
+
FileOpener *opener) {
|
93
|
+
return FindFileSystem(directory).ListFiles(directory, callback, opener);
|
94
|
+
}
|
95
|
+
|
96
|
+
void VirtualFileSystem::MoveFile(const string &source, const string &target) {
|
97
|
+
FindFileSystem(source).MoveFile(source, target);
|
98
|
+
}
|
99
|
+
|
100
|
+
bool VirtualFileSystem::FileExists(const string &filename) {
|
101
|
+
return FindFileSystem(filename).FileExists(filename);
|
102
|
+
}
|
103
|
+
|
104
|
+
bool VirtualFileSystem::IsPipe(const string &filename) {
|
105
|
+
return FindFileSystem(filename).IsPipe(filename);
|
106
|
+
}
|
107
|
+
void VirtualFileSystem::RemoveFile(const string &filename) {
|
108
|
+
FindFileSystem(filename).RemoveFile(filename);
|
109
|
+
}
|
110
|
+
|
111
|
+
vector<string> VirtualFileSystem::Glob(const string &path, FileOpener *opener) {
|
112
|
+
return FindFileSystem(path).Glob(path, opener);
|
113
|
+
}
|
114
|
+
|
115
|
+
void VirtualFileSystem::RegisterSubSystem(unique_ptr<FileSystem> fs) {
|
116
|
+
sub_systems.push_back(std::move(fs));
|
117
|
+
}
|
118
|
+
|
119
|
+
void VirtualFileSystem::UnregisterSubSystem(const string &name) {
|
120
|
+
for (auto sub_system = sub_systems.begin(); sub_system != sub_systems.end(); sub_system++) {
|
121
|
+
if (sub_system->get()->GetName() == name) {
|
122
|
+
sub_systems.erase(sub_system);
|
123
|
+
return;
|
124
|
+
}
|
125
|
+
}
|
126
|
+
throw InvalidInputException("Could not find filesystem with name %s", name);
|
127
|
+
}
|
128
|
+
|
129
|
+
void VirtualFileSystem::RegisterSubSystem(FileCompressionType compression_type, unique_ptr<FileSystem> fs) {
|
130
|
+
compressed_fs[compression_type] = std::move(fs);
|
131
|
+
}
|
132
|
+
|
133
|
+
vector<string> VirtualFileSystem::ListSubSystems() {
|
134
|
+
vector<string> names(sub_systems.size());
|
135
|
+
for (idx_t i = 0; i < sub_systems.size(); i++) {
|
136
|
+
names[i] = sub_systems[i]->GetName();
|
137
|
+
}
|
138
|
+
return names;
|
139
|
+
}
|
140
|
+
|
141
|
+
std::string VirtualFileSystem::GetName() const {
|
142
|
+
return "VirtualFileSystem";
|
143
|
+
}
|
144
|
+
|
145
|
+
void VirtualFileSystem::SetDisabledFileSystems(const vector<string> &names) {
|
146
|
+
unordered_set<string> new_disabled_file_systems;
|
147
|
+
for (auto &name : names) {
|
148
|
+
if (name.empty()) {
|
149
|
+
continue;
|
150
|
+
}
|
151
|
+
if (new_disabled_file_systems.find(name) != new_disabled_file_systems.end()) {
|
152
|
+
throw InvalidInputException("Duplicate disabled file system \"%s\"", name);
|
153
|
+
}
|
154
|
+
new_disabled_file_systems.insert(name);
|
155
|
+
}
|
156
|
+
for (auto &disabled_fs : disabled_file_systems) {
|
157
|
+
if (new_disabled_file_systems.find(disabled_fs) == new_disabled_file_systems.end()) {
|
158
|
+
throw InvalidInputException("File system \"%s\" has been disabled previously, it cannot be re-enabled",
|
159
|
+
disabled_fs);
|
160
|
+
}
|
161
|
+
}
|
162
|
+
disabled_file_systems = std::move(new_disabled_file_systems);
|
163
|
+
}
|
164
|
+
|
165
|
+
FileSystem &VirtualFileSystem::FindFileSystem(const string &path) {
|
166
|
+
auto &fs = FindFileSystemInternal(path);
|
167
|
+
if (!disabled_file_systems.empty() && disabled_file_systems.find(fs.GetName()) != disabled_file_systems.end()) {
|
168
|
+
throw PermissionException("File system %s has been disabled by configuration", fs.GetName());
|
169
|
+
}
|
170
|
+
return fs;
|
171
|
+
}
|
172
|
+
|
173
|
+
FileSystem &VirtualFileSystem::FindFileSystemInternal(const string &path) {
|
174
|
+
for (auto &sub_system : sub_systems) {
|
175
|
+
if (sub_system->CanHandleFile(path)) {
|
176
|
+
return *sub_system;
|
177
|
+
}
|
178
|
+
}
|
179
|
+
return *default_fs;
|
180
|
+
}
|
181
|
+
|
45
182
|
} // namespace duckdb
|
@@ -137,9 +137,9 @@ static StaticFunctionDefinition internal_functions[] = {
|
|
137
137
|
DUCKDB_SCALAR_FUNCTION(EnumRangeFun),
|
138
138
|
DUCKDB_SCALAR_FUNCTION(EnumRangeBoundaryFun),
|
139
139
|
DUCKDB_SCALAR_FUNCTION_SET(EpochFun),
|
140
|
-
DUCKDB_SCALAR_FUNCTION_SET(EpochNanosecondsFun),
|
141
|
-
DUCKDB_SCALAR_FUNCTION_SET(EpochMicrosecondsFun),
|
142
140
|
DUCKDB_SCALAR_FUNCTION_SET(EpochMsFun),
|
141
|
+
DUCKDB_SCALAR_FUNCTION_SET(EpochNsFun),
|
142
|
+
DUCKDB_SCALAR_FUNCTION_SET(EpochUsFun),
|
143
143
|
DUCKDB_SCALAR_FUNCTION_SET(EraFun),
|
144
144
|
DUCKDB_SCALAR_FUNCTION(ErrorFun),
|
145
145
|
DUCKDB_SCALAR_FUNCTION(EvenFun),
|
@@ -1591,7 +1591,7 @@ ScalarFunctionSet EpochFun::GetFunctions() {
|
|
1591
1591
|
return GetTimePartFunction<DatePart::EpochOperator>();
|
1592
1592
|
}
|
1593
1593
|
|
1594
|
-
ScalarFunctionSet
|
1594
|
+
ScalarFunctionSet EpochNsFun::GetFunctions() {
|
1595
1595
|
using OP = DatePart::EpochNanosecondsOperator;
|
1596
1596
|
auto operator_set = GetTimePartFunction<OP>();
|
1597
1597
|
|
@@ -1603,7 +1603,7 @@ ScalarFunctionSet EpochNanosecondsFun::GetFunctions() {
|
|
1603
1603
|
return operator_set;
|
1604
1604
|
}
|
1605
1605
|
|
1606
|
-
ScalarFunctionSet
|
1606
|
+
ScalarFunctionSet EpochUsFun::GetFunctions() {
|
1607
1607
|
using OP = DatePart::EpochMicrosecondsOperator;
|
1608
1608
|
auto operator_set = GetTimePartFunction<OP>();
|
1609
1609
|
|
@@ -1,7 +1,6 @@
|
|
1
1
|
#include "duckdb/core_functions/scalar/date_functions.hpp"
|
2
|
-
|
3
|
-
#include "duckdb/common/
|
4
|
-
#include "duckdb/common/types/timestamp.hpp"
|
2
|
+
|
3
|
+
#include "duckdb/common/operator/cast_operators.hpp"
|
5
4
|
#include "duckdb/common/vector_operations/vector_operations.hpp"
|
6
5
|
#include "duckdb/common/vector_operations/unary_executor.hpp"
|
7
6
|
|
@@ -9,20 +8,24 @@ namespace duckdb {
|
|
9
8
|
|
10
9
|
struct EpochSecOperator {
|
11
10
|
template <class INPUT_TYPE, class RESULT_TYPE>
|
12
|
-
static RESULT_TYPE Operation(INPUT_TYPE
|
13
|
-
|
11
|
+
static RESULT_TYPE Operation(INPUT_TYPE sec) {
|
12
|
+
int64_t result;
|
13
|
+
if (!TryCast::Operation(sec * Interval::MICROS_PER_SEC, result)) {
|
14
|
+
throw ConversionException("Could not convert epoch seconds to TIMESTAMP WITH TIME ZONE");
|
15
|
+
}
|
16
|
+
return timestamp_t(result);
|
14
17
|
}
|
15
18
|
};
|
16
19
|
|
17
20
|
static void EpochSecFunction(DataChunk &input, ExpressionState &state, Vector &result) {
|
18
21
|
D_ASSERT(input.ColumnCount() == 1);
|
19
22
|
|
20
|
-
UnaryExecutor::Execute<
|
23
|
+
UnaryExecutor::Execute<double, timestamp_t, EpochSecOperator>(input.data[0], result, input.size());
|
21
24
|
}
|
22
25
|
|
23
26
|
ScalarFunction ToTimestampFun::GetFunction() {
|
24
27
|
// to_timestamp is an alias from Postgres that converts the time in seconds to a timestamp
|
25
|
-
return ScalarFunction({LogicalType::
|
28
|
+
return ScalarFunction({LogicalType::DOUBLE}, LogicalType::TIMESTAMP_TZ, EpochSecFunction);
|
26
29
|
}
|
27
30
|
|
28
31
|
} // namespace duckdb
|
@@ -16,17 +16,18 @@ PhysicalTableScan::PhysicalTableScan(vector<LogicalType> types, TableFunction fu
|
|
16
16
|
: PhysicalOperator(PhysicalOperatorType::TABLE_SCAN, std::move(types), estimated_cardinality),
|
17
17
|
function(std::move(function_p)), bind_data(std::move(bind_data_p)), column_ids(std::move(column_ids_p)),
|
18
18
|
names(std::move(names_p)), table_filters(std::move(table_filters_p)) {
|
19
|
+
extra_info.file_filters = "";
|
19
20
|
}
|
20
21
|
|
21
22
|
PhysicalTableScan::PhysicalTableScan(vector<LogicalType> types, TableFunction function_p,
|
22
23
|
unique_ptr<FunctionData> bind_data_p, vector<LogicalType> returned_types_p,
|
23
24
|
vector<column_t> column_ids_p, vector<idx_t> projection_ids_p,
|
24
25
|
vector<string> names_p, unique_ptr<TableFilterSet> table_filters_p,
|
25
|
-
idx_t estimated_cardinality)
|
26
|
+
idx_t estimated_cardinality, ExtraOperatorInfo extra_info)
|
26
27
|
: PhysicalOperator(PhysicalOperatorType::TABLE_SCAN, std::move(types), estimated_cardinality),
|
27
28
|
function(std::move(function_p)), bind_data(std::move(bind_data_p)), returned_types(std::move(returned_types_p)),
|
28
29
|
column_ids(std::move(column_ids_p)), projection_ids(std::move(projection_ids_p)), names(std::move(names_p)),
|
29
|
-
table_filters(std::move(table_filters_p)) {
|
30
|
+
table_filters(std::move(table_filters_p)), extra_info(extra_info) {
|
30
31
|
}
|
31
32
|
|
32
33
|
class TableScanGlobalSourceState : public GlobalSourceState {
|
@@ -149,6 +150,10 @@ string PhysicalTableScan::ParamsToString() const {
|
|
149
150
|
}
|
150
151
|
}
|
151
152
|
}
|
153
|
+
if (!extra_info.file_filters.empty()) {
|
154
|
+
result += "\n[INFOSEPARATOR]\n";
|
155
|
+
result += "File Filters: " + extra_info.file_filters;
|
156
|
+
}
|
152
157
|
result += "\n[INFOSEPARATOR]\n";
|
153
158
|
result += StringUtil::Format("EC: %llu", estimated_props->GetCardinality<idx_t>());
|
154
159
|
return result;
|
@@ -55,7 +55,7 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalGet &op) {
|
|
55
55
|
// function does not support projection pushdown
|
56
56
|
auto node = make_uniq<PhysicalTableScan>(op.returned_types, op.function, std::move(op.bind_data),
|
57
57
|
op.returned_types, op.column_ids, vector<column_t>(), op.names,
|
58
|
-
std::move(table_filters), op.estimated_cardinality);
|
58
|
+
std::move(table_filters), op.estimated_cardinality, op.extra_info);
|
59
59
|
// first check if an additional projection is necessary
|
60
60
|
if (op.column_ids.size() == op.returned_types.size()) {
|
61
61
|
bool projection_necessary = false;
|
@@ -93,7 +93,7 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalGet &op) {
|
|
93
93
|
} else {
|
94
94
|
return make_uniq<PhysicalTableScan>(op.types, op.function, std::move(op.bind_data), op.returned_types,
|
95
95
|
op.column_ids, op.projection_ids, op.names, std::move(table_filters),
|
96
|
-
op.estimated_cardinality);
|
96
|
+
op.estimated_cardinality, op.extra_info);
|
97
97
|
}
|
98
98
|
}
|
99
99
|
|
@@ -1,3 +1,6 @@
|
|
1
|
+
#include "duckdb/common/hugeint.hpp"
|
2
|
+
#include "duckdb/common/operator/cast_operators.hpp"
|
3
|
+
#include "duckdb/common/types.hpp"
|
1
4
|
#include "duckdb/function/cast/default_casts.hpp"
|
2
5
|
#include "duckdb/function/cast/vector_cast_helpers.hpp"
|
3
6
|
|
@@ -6,9 +9,38 @@ namespace duckdb {
|
|
6
9
|
BoundCastInfo DefaultCasts::BitCastSwitch(BindCastInput &input, const LogicalType &source, const LogicalType &target) {
|
7
10
|
// now switch on the result type
|
8
11
|
switch (target.id()) {
|
12
|
+
// Numerics
|
13
|
+
case LogicalTypeId::BOOLEAN:
|
14
|
+
return BoundCastInfo(&VectorCastHelpers::TryCastLoop<string_t, bool, CastFromBitToNumeric>);
|
15
|
+
case LogicalTypeId::TINYINT:
|
16
|
+
return BoundCastInfo(&VectorCastHelpers::TryCastLoop<string_t, int8_t, CastFromBitToNumeric>);
|
17
|
+
case LogicalTypeId::SMALLINT:
|
18
|
+
return BoundCastInfo(&VectorCastHelpers::TryCastLoop<string_t, int16_t, CastFromBitToNumeric>);
|
19
|
+
case LogicalTypeId::INTEGER:
|
20
|
+
return BoundCastInfo(&VectorCastHelpers::TryCastLoop<string_t, int32_t, CastFromBitToNumeric>);
|
21
|
+
case LogicalTypeId::BIGINT:
|
22
|
+
return BoundCastInfo(&VectorCastHelpers::TryCastLoop<string_t, int64_t, CastFromBitToNumeric>);
|
23
|
+
case LogicalTypeId::UTINYINT:
|
24
|
+
return BoundCastInfo(&VectorCastHelpers::TryCastLoop<string_t, uint8_t, CastFromBitToNumeric>);
|
25
|
+
case LogicalTypeId::USMALLINT:
|
26
|
+
return BoundCastInfo(&VectorCastHelpers::TryCastLoop<string_t, uint16_t, CastFromBitToNumeric>);
|
27
|
+
case LogicalTypeId::UINTEGER:
|
28
|
+
return BoundCastInfo(&VectorCastHelpers::TryCastLoop<string_t, uint32_t, CastFromBitToNumeric>);
|
29
|
+
case LogicalTypeId::UBIGINT:
|
30
|
+
return BoundCastInfo(&VectorCastHelpers::TryCastLoop<string_t, uint64_t, CastFromBitToNumeric>);
|
31
|
+
case LogicalTypeId::HUGEINT:
|
32
|
+
return BoundCastInfo(&VectorCastHelpers::TryCastLoop<string_t, hugeint_t, CastFromBitToNumeric>);
|
33
|
+
case LogicalTypeId::FLOAT:
|
34
|
+
return BoundCastInfo(&VectorCastHelpers::TryCastLoop<string_t, float, CastFromBitToNumeric>);
|
35
|
+
case LogicalTypeId::DOUBLE:
|
36
|
+
return BoundCastInfo(&VectorCastHelpers::TryCastLoop<string_t, double, CastFromBitToNumeric>);
|
37
|
+
|
38
|
+
case LogicalTypeId::BLOB:
|
39
|
+
return BoundCastInfo(&VectorCastHelpers::StringCast<string_t, CastFromBitToBlob>);
|
40
|
+
|
9
41
|
case LogicalTypeId::VARCHAR:
|
10
|
-
|
11
|
-
|
42
|
+
return BoundCastInfo(&VectorCastHelpers::StringCast<string_t, CastFromBitToString>);
|
43
|
+
|
12
44
|
default:
|
13
45
|
return DefaultCasts::TryVectorNullCast;
|
14
46
|
}
|
@@ -11,6 +11,9 @@ BoundCastInfo DefaultCasts::BlobCastSwitch(BindCastInput &input, const LogicalTy
|
|
11
11
|
return BoundCastInfo(&VectorCastHelpers::StringCast<string_t, duckdb::CastFromBlob>);
|
12
12
|
case LogicalTypeId::AGGREGATE_STATE:
|
13
13
|
return DefaultCasts::ReinterpretCast;
|
14
|
+
case LogicalTypeId::BIT:
|
15
|
+
return BoundCastInfo(&VectorCastHelpers::StringCast<string_t, duckdb::CastFromBlobToBit>);
|
16
|
+
|
14
17
|
default:
|
15
18
|
return DefaultCasts::TryVectorNullCast;
|
16
19
|
}
|
@@ -37,6 +37,8 @@ static BoundCastInfo InternalNumericCastSwitch(const LogicalType &source, const
|
|
37
37
|
return BoundCastInfo(&VectorCastHelpers::ToDecimalCast<SRC>);
|
38
38
|
case LogicalTypeId::VARCHAR:
|
39
39
|
return BoundCastInfo(&VectorCastHelpers::StringCast<SRC, duckdb::StringCast>);
|
40
|
+
case LogicalTypeId::BIT:
|
41
|
+
return BoundCastInfo(&VectorCastHelpers::StringCast<SRC, duckdb::NumericTryCastToBit>);
|
40
42
|
default:
|
41
43
|
return DefaultCasts::TryVectorNullCast;
|
42
44
|
}
|
@@ -1,8 +1,8 @@
|
|
1
1
|
#ifndef DUCKDB_VERSION
|
2
|
-
#define DUCKDB_VERSION "0.8.2-
|
2
|
+
#define DUCKDB_VERSION "0.8.2-dev1435"
|
3
3
|
#endif
|
4
4
|
#ifndef DUCKDB_SOURCE_ID
|
5
|
-
#define DUCKDB_SOURCE_ID "
|
5
|
+
#define DUCKDB_SOURCE_ID "50d861cee9"
|
6
6
|
#endif
|
7
7
|
#include "duckdb/function/table/system_functions.hpp"
|
8
8
|
#include "duckdb/main/database.hpp"
|
@@ -0,0 +1,27 @@
|
|
1
|
+
//===----------------------------------------------------------------------===//
|
2
|
+
// DuckDB
|
3
|
+
//
|
4
|
+
// duckdb/common/extra_operator_info.hpp
|
5
|
+
//
|
6
|
+
//
|
7
|
+
//===----------------------------------------------------------------------===//
|
8
|
+
|
9
|
+
#pragma once
|
10
|
+
|
11
|
+
#include <algorithm>
|
12
|
+
#include <cstdint>
|
13
|
+
#include <cstring>
|
14
|
+
#include "duckdb/common/operator/comparison_operators.hpp"
|
15
|
+
|
16
|
+
namespace duckdb {
|
17
|
+
|
18
|
+
class ExtraOperatorInfo {
|
19
|
+
public:
|
20
|
+
ExtraOperatorInfo() : file_filters("") {
|
21
|
+
}
|
22
|
+
ExtraOperatorInfo(ExtraOperatorInfo &extra_info) : file_filters(extra_info.file_filters) {
|
23
|
+
}
|
24
|
+
string file_filters;
|
25
|
+
};
|
26
|
+
|
27
|
+
} // namespace duckdb
|
@@ -246,6 +246,8 @@ public:
|
|
246
246
|
|
247
247
|
//! Whether or not a file is remote or local, based only on file path
|
248
248
|
DUCKDB_API static bool IsRemoteFile(const string &path);
|
249
|
+
|
250
|
+
DUCKDB_API virtual void SetDisabledFileSystems(const vector<string> &names);
|
249
251
|
};
|
250
252
|
|
251
253
|
} // namespace duckdb
|
@@ -31,7 +31,7 @@ public:
|
|
31
31
|
//! evaluate to true.
|
32
32
|
DUCKDB_API static void ApplyFiltersToFileList(ClientContext &context, vector<string> &files,
|
33
33
|
vector<unique_ptr<Expression>> &filters,
|
34
|
-
unordered_map<string, column_t> &column_map,
|
34
|
+
unordered_map<string, column_t> &column_map, LogicalGet &get,
|
35
35
|
bool hive_enabled, bool filename_enabled);
|
36
36
|
|
37
37
|
//! Returns the compiled regex pattern to match hive partitions
|