duckdb 0.8.2-dev1182.0 → 0.8.2-dev1435.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/binding.gyp +7 -7
- package/package.json +1 -1
- package/src/duckdb/extension/parquet/parquet_extension.cpp +1 -1
- package/src/duckdb/extension/parquet/parquet_reader.cpp +1 -1
- package/src/duckdb/src/common/arrow/arrow_appender.cpp +0 -1
- package/src/duckdb/src/common/exception.cpp +2 -2
- package/src/duckdb/src/common/file_system.cpp +4 -0
- package/src/duckdb/src/common/hive_partitioning.cpp +10 -6
- package/src/duckdb/src/common/local_file_system.cpp +1 -1
- package/src/duckdb/src/common/multi_file_reader.cpp +3 -2
- package/src/duckdb/src/common/operator/cast_operators.cpp +35 -1
- package/src/duckdb/src/common/re2_regex.cpp +1 -1
- package/src/duckdb/src/common/types/bit.cpp +51 -0
- package/src/duckdb/src/common/types/uuid.cpp +2 -2
- package/src/duckdb/src/common/virtual_file_system.cpp +138 -1
- package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp +2 -0
- package/src/duckdb/src/core_functions/function_list.cpp +2 -2
- package/src/duckdb/src/core_functions/scalar/date/date_part.cpp +2 -2
- package/src/duckdb/src/core_functions/scalar/date/epoch.cpp +10 -7
- package/src/duckdb/src/execution/expression_executor.cpp +1 -1
- package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +3 -3
- package/src/duckdb/src/execution/operator/join/physical_asof_join.cpp +1 -1
- package/src/duckdb/src/execution/operator/join/physical_range_join.cpp +2 -2
- package/src/duckdb/src/execution/operator/persistent/csv_file_handle.cpp +1 -1
- package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +1 -1
- package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp +7 -2
- package/src/duckdb/src/execution/physical_plan/plan_get.cpp +2 -2
- package/src/duckdb/src/execution/window_segment_tree.cpp +1 -1
- package/src/duckdb/src/function/cast/bit_cast.cpp +34 -2
- package/src/duckdb/src/function/cast/blob_cast.cpp +3 -0
- package/src/duckdb/src/function/cast/numeric_casts.cpp +2 -0
- package/src/duckdb/src/function/function.cpp +1 -1
- package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
- package/src/duckdb/src/include/duckdb/common/extra_operator_info.hpp +27 -0
- package/src/duckdb/src/include/duckdb/common/file_system.hpp +2 -0
- package/src/duckdb/src/include/duckdb/common/hive_partitioning.hpp +1 -1
- package/src/duckdb/src/include/duckdb/common/operator/cast_operators.hpp +43 -3
- package/src/duckdb/src/include/duckdb/common/operator/numeric_cast.hpp +10 -0
- package/src/duckdb/src/include/duckdb/common/types/bit.hpp +81 -0
- package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp +38 -97
- package/src/duckdb/src/include/duckdb/core_functions/scalar/date_functions.hpp +11 -11
- package/src/duckdb/src/include/duckdb/execution/operator/persistent/base_csv_reader.hpp +1 -1
- package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp +5 -1
- package/src/duckdb/src/include/duckdb/main/connection.hpp +1 -2
- package/src/duckdb/src/include/duckdb/main/relation/cross_product_relation.hpp +4 -1
- package/src/duckdb/src/include/duckdb/main/relation/join_relation.hpp +5 -2
- package/src/duckdb/src/include/duckdb/main/relation.hpp +4 -2
- package/src/duckdb/src/include/duckdb/main/settings.hpp +9 -0
- package/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp +1 -1
- package/src/duckdb/src/include/duckdb/planner/constraints/bound_unique_constraint.hpp +3 -3
- package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp +4 -0
- package/src/duckdb/src/main/capi/duckdb_value-c.cpp +1 -1
- package/src/duckdb/src/main/config.cpp +1 -0
- package/src/duckdb/src/main/database.cpp +1 -1
- package/src/duckdb/src/main/extension/extension_install.cpp +6 -0
- package/src/duckdb/src/main/extension/extension_load.cpp +10 -1
- package/src/duckdb/src/main/relation/cross_product_relation.cpp +4 -3
- package/src/duckdb/src/main/relation/join_relation.cpp +5 -5
- package/src/duckdb/src/main/relation.cpp +6 -5
- package/src/duckdb/src/main/settings/settings.cpp +24 -0
- package/src/duckdb/src/parser/parser.cpp +8 -2
- package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +5 -4
- package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp +1 -1
- package/src/duckdb/src/planner/operator/logical_get.cpp +9 -4
- package/src/duckdb/src/storage/compression/rle.cpp +0 -1
- package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +6 -6
- package/src/statement.cpp +2 -1
package/binding.gyp
CHANGED
@@ -239,18 +239,18 @@
|
|
239
239
|
"src/duckdb/third_party/zstd/compress/zstd_lazy.cpp",
|
240
240
|
"src/duckdb/third_party/zstd/compress/zstd_ldm.cpp",
|
241
241
|
"src/duckdb/third_party/zstd/compress/zstd_opt.cpp",
|
242
|
-
"src/duckdb/extension/icu/./icu-datepart.cpp",
|
243
|
-
"src/duckdb/extension/icu/./icu-timezone.cpp",
|
244
|
-
"src/duckdb/extension/icu/./icu-makedate.cpp",
|
245
|
-
"src/duckdb/extension/icu/./icu-table-range.cpp",
|
246
|
-
"src/duckdb/extension/icu/./icu-datefunc.cpp",
|
247
242
|
"src/duckdb/extension/icu/./icu-list-range.cpp",
|
248
|
-
"src/duckdb/extension/icu/./icu-
|
243
|
+
"src/duckdb/extension/icu/./icu-table-range.cpp",
|
249
244
|
"src/duckdb/extension/icu/./icu-datesub.cpp",
|
245
|
+
"src/duckdb/extension/icu/./icu-dateadd.cpp",
|
250
246
|
"src/duckdb/extension/icu/./icu-timebucket.cpp",
|
251
|
-
"src/duckdb/extension/icu/./icu-
|
247
|
+
"src/duckdb/extension/icu/./icu-makedate.cpp",
|
248
|
+
"src/duckdb/extension/icu/./icu-timezone.cpp",
|
249
|
+
"src/duckdb/extension/icu/./icu-datefunc.cpp",
|
250
|
+
"src/duckdb/extension/icu/./icu-datepart.cpp",
|
252
251
|
"src/duckdb/extension/icu/./icu_extension.cpp",
|
253
252
|
"src/duckdb/extension/icu/./icu-datetrunc.cpp",
|
253
|
+
"src/duckdb/extension/icu/./icu-strptime.cpp",
|
254
254
|
"src/duckdb/ub_extension_icu_third_party_icu_common.cpp",
|
255
255
|
"src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp",
|
256
256
|
"src/duckdb/extension/icu/third_party/icu/stubdata/stubdata.cpp",
|
package/package.json
CHANGED
@@ -175,7 +175,6 @@ public:
|
|
175
175
|
table_function.serialize = ParquetScanSerialize;
|
176
176
|
table_function.deserialize = ParquetScanDeserialize;
|
177
177
|
table_function.get_batch_info = ParquetGetBatchInfo;
|
178
|
-
|
179
178
|
table_function.projection_pushdown = true;
|
180
179
|
table_function.filter_pushdown = true;
|
181
180
|
table_function.filter_prune = true;
|
@@ -520,6 +519,7 @@ public:
|
|
520
519
|
static void ParquetComplexFilterPushdown(ClientContext &context, LogicalGet &get, FunctionData *bind_data_p,
|
521
520
|
vector<unique_ptr<Expression>> &filters) {
|
522
521
|
auto &data = bind_data_p->Cast<ParquetReadBindData>();
|
522
|
+
|
523
523
|
auto reset_reader = MultiFileReader::ComplexFilterPushdown(context, data.files,
|
524
524
|
data.parquet_options.file_options, get, filters);
|
525
525
|
if (reset_reader) {
|
@@ -76,7 +76,7 @@ static shared_ptr<ParquetFileMetadataCache> LoadMetadata(Allocator &allocator, F
|
|
76
76
|
}
|
77
77
|
// read four-byte footer length from just before the end magic bytes
|
78
78
|
auto footer_len = *reinterpret_cast<uint32_t *>(buf.ptr);
|
79
|
-
if (footer_len
|
79
|
+
if (footer_len == 0 || file_size < 12 + footer_len) {
|
80
80
|
throw InvalidInputException("Footer length error in file '%s'", file_handle.path);
|
81
81
|
}
|
82
82
|
auto metadata_pos = file_size - (footer_len + 8);
|
@@ -786,7 +786,6 @@ ArrowArray ArrowAppender::Finalize() {
|
|
786
786
|
|
787
787
|
// Configure root array
|
788
788
|
result.length = row_count;
|
789
|
-
result.n_children = types.size();
|
790
789
|
result.n_buffers = 1;
|
791
790
|
result.buffers = root_holder->buffers.data(); // there is no actual buffer there since we don't have NULLs
|
792
791
|
result.offset = 0;
|
@@ -62,11 +62,11 @@ string Exception::ConstructMessageRecursive(const string &msg, std::vector<Excep
|
|
62
62
|
#ifdef DEBUG
|
63
63
|
// Verify that we have the required amount of values for the message
|
64
64
|
idx_t parameter_count = 0;
|
65
|
-
for (idx_t i = 0; i < msg.size(); i++) {
|
65
|
+
for (idx_t i = 0; i + 1 < msg.size(); i++) {
|
66
66
|
if (msg[i] != '%') {
|
67
67
|
continue;
|
68
68
|
}
|
69
|
-
if (
|
69
|
+
if (msg[i + 1] == '%') {
|
70
70
|
i++;
|
71
71
|
continue;
|
72
72
|
}
|
@@ -385,6 +385,10 @@ void FileSystem::UnregisterSubSystem(const string &name) {
|
|
385
385
|
throw NotImplementedException("%s: Can't unregister a sub system on a non-virtual file system", GetName());
|
386
386
|
}
|
387
387
|
|
388
|
+
void FileSystem::SetDisabledFileSystems(const vector<string> &names) {
|
389
|
+
throw NotImplementedException("%s: Can't disable file systems on a non-virtual file system", GetName());
|
390
|
+
}
|
391
|
+
|
388
392
|
vector<string> FileSystem::ListSubSystems() {
|
389
393
|
throw NotImplementedException("%s: Can't list sub systems on a non-virtual file system", GetName());
|
390
394
|
}
|
@@ -6,6 +6,7 @@
|
|
6
6
|
#include "duckdb/planner/expression/bound_constant_expression.hpp"
|
7
7
|
#include "duckdb/planner/expression/bound_reference_expression.hpp"
|
8
8
|
#include "duckdb/planner/expression_iterator.hpp"
|
9
|
+
#include "duckdb/planner/operator/logical_get.hpp"
|
9
10
|
#include "duckdb/planner/table_filter.hpp"
|
10
11
|
#include "re2/re2.h"
|
11
12
|
|
@@ -86,12 +87,15 @@ std::map<string, string> HivePartitioning::Parse(const string &filename) {
|
|
86
87
|
// currently, only expressions that cannot be evaluated during pushdown are removed.
|
87
88
|
void HivePartitioning::ApplyFiltersToFileList(ClientContext &context, vector<string> &files,
|
88
89
|
vector<unique_ptr<Expression>> &filters,
|
89
|
-
unordered_map<string, column_t> &column_map,
|
90
|
+
unordered_map<string, column_t> &column_map, LogicalGet &get,
|
90
91
|
bool hive_enabled, bool filename_enabled) {
|
92
|
+
|
91
93
|
vector<string> pruned_files;
|
92
94
|
vector<bool> have_preserved_filter(filters.size(), false);
|
93
95
|
vector<unique_ptr<Expression>> pruned_filters;
|
96
|
+
unordered_set<idx_t> filters_applied_to_files;
|
94
97
|
duckdb_re2::RE2 regex(REGEX_STRING);
|
98
|
+
auto table_index = get.table_index;
|
95
99
|
|
96
100
|
if ((!filename_enabled && !hive_enabled) || filters.empty()) {
|
97
101
|
return;
|
@@ -121,11 +125,11 @@ void HivePartitioning::ApplyFiltersToFileList(ClientContext &context, vector<str
|
|
121
125
|
} else if (!result_value.GetValue<bool>()) {
|
122
126
|
// filter evaluates to false
|
123
127
|
should_prune_file = true;
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
128
|
+
// convert the filter to a table filter.
|
129
|
+
if (filters_applied_to_files.find(j) == filters_applied_to_files.end()) {
|
130
|
+
get.extra_info.file_filters += filter->ToString();
|
131
|
+
filters_applied_to_files.insert(j);
|
132
|
+
}
|
129
133
|
}
|
130
134
|
}
|
131
135
|
|
@@ -375,7 +375,7 @@ int RemoveDirectoryRecursive(const char *path) {
|
|
375
375
|
continue;
|
376
376
|
}
|
377
377
|
len = path_len + (idx_t)strlen(p->d_name) + 2;
|
378
|
-
buf = new char[len];
|
378
|
+
buf = new (std::nothrow) char[len];
|
379
379
|
if (buf) {
|
380
380
|
struct stat statbuf;
|
381
381
|
snprintf(buf, len, "%s/%s", path, p->d_name);
|
@@ -106,8 +106,9 @@ bool MultiFileReader::ComplexFilterPushdown(ClientContext &context, vector<strin
|
|
106
106
|
}
|
107
107
|
|
108
108
|
auto start_files = files.size();
|
109
|
-
HivePartitioning::ApplyFiltersToFileList(context, files, filters, column_map, get.
|
110
|
-
options.
|
109
|
+
HivePartitioning::ApplyFiltersToFileList(context, files, filters, column_map, get, options.hive_partitioning,
|
110
|
+
options.filename);
|
111
|
+
|
111
112
|
if (files.size() != start_files) {
|
112
113
|
// we have pruned files
|
113
114
|
return true;
|
@@ -1,4 +1,5 @@
|
|
1
1
|
#include "duckdb/common/operator/cast_operators.hpp"
|
2
|
+
#include "duckdb/common/hugeint.hpp"
|
2
3
|
#include "duckdb/common/operator/string_cast.hpp"
|
3
4
|
#include "duckdb/common/operator/numeric_cast.hpp"
|
4
5
|
#include "duckdb/common/operator/decimal_cast_operators.hpp"
|
@@ -1425,11 +1426,20 @@ string_t CastFromBlob::Operation(string_t input, Vector &vector) {
|
|
1425
1426
|
return result;
|
1426
1427
|
}
|
1427
1428
|
|
1429
|
+
template <>
|
1430
|
+
string_t CastFromBlobToBit::Operation(string_t input, Vector &vector) {
|
1431
|
+
idx_t result_size = input.GetSize() + 1;
|
1432
|
+
if (result_size <= 1) {
|
1433
|
+
throw ConversionException("Cannot cast empty BLOB to BIT");
|
1434
|
+
}
|
1435
|
+
return StringVector::AddStringOrBlob(vector, Bit::BlobToBit(input));
|
1436
|
+
}
|
1437
|
+
|
1428
1438
|
//===--------------------------------------------------------------------===//
|
1429
1439
|
// Cast From Bit
|
1430
1440
|
//===--------------------------------------------------------------------===//
|
1431
1441
|
template <>
|
1432
|
-
string_t
|
1442
|
+
string_t CastFromBitToString::Operation(string_t input, Vector &vector) {
|
1433
1443
|
|
1434
1444
|
idx_t result_size = Bit::BitLength(input);
|
1435
1445
|
string_t result = StringVector::EmptyString(vector, result_size);
|
@@ -1482,6 +1492,30 @@ bool TryCastToBit::Operation(string_t input, string_t &result, Vector &result_ve
|
|
1482
1492
|
return true;
|
1483
1493
|
}
|
1484
1494
|
|
1495
|
+
template <>
|
1496
|
+
bool CastFromBitToNumeric::Operation(string_t input, bool &result, bool strict) {
|
1497
|
+
D_ASSERT(input.GetSize() > 1);
|
1498
|
+
|
1499
|
+
uint8_t value;
|
1500
|
+
bool success = CastFromBitToNumeric::Operation(input, value, strict);
|
1501
|
+
result = (value > 0);
|
1502
|
+
return (success);
|
1503
|
+
}
|
1504
|
+
|
1505
|
+
template <>
|
1506
|
+
bool CastFromBitToNumeric::Operation(string_t input, hugeint_t &result, bool strict) {
|
1507
|
+
D_ASSERT(input.GetSize() > 1);
|
1508
|
+
|
1509
|
+
if (input.GetSize() - 1 > sizeof(hugeint_t)) {
|
1510
|
+
throw ConversionException("Bitstring doesn't fit inside of %s", GetTypeId<hugeint_t>());
|
1511
|
+
}
|
1512
|
+
Bit::BitToNumeric(input, result);
|
1513
|
+
if (result < NumericLimits<hugeint_t>::Minimum()) {
|
1514
|
+
throw ConversionException("Minimum limit for HUGEINT is %s", NumericLimits<hugeint_t>::Minimum().ToString());
|
1515
|
+
}
|
1516
|
+
return (true);
|
1517
|
+
}
|
1518
|
+
|
1485
1519
|
//===--------------------------------------------------------------------===//
|
1486
1520
|
// Cast From UUID
|
1487
1521
|
//===--------------------------------------------------------------------===//
|
@@ -54,7 +54,7 @@ duckdb::vector<Match> RegexFindAll(const std::string &input, const Regex ®ex)
|
|
54
54
|
Match match;
|
55
55
|
while (RegexSearchInternal(input.c_str(), match, regex, RE2::UNANCHORED, position, input.size())) {
|
56
56
|
position += match.position(0) + match.length(0);
|
57
|
-
matches.emplace_back(
|
57
|
+
matches.emplace_back(match);
|
58
58
|
}
|
59
59
|
return matches;
|
60
60
|
}
|
@@ -1,4 +1,6 @@
|
|
1
|
+
#include "duckdb/common/assert.hpp"
|
1
2
|
#include "duckdb/common/operator/cast_operators.hpp"
|
3
|
+
#include "duckdb/common/typedefs.hpp"
|
2
4
|
#include "duckdb/common/types/bit.hpp"
|
3
5
|
#include "duckdb/common/types/string_type.hpp"
|
4
6
|
|
@@ -34,6 +36,13 @@ static inline idx_t GetBitSize(const string_t &str) {
|
|
34
36
|
return str_len;
|
35
37
|
}
|
36
38
|
|
39
|
+
uint8_t Bit::GetFirstByte(const string_t &str) {
|
40
|
+
D_ASSERT(str.GetSize() > 1);
|
41
|
+
|
42
|
+
auto data = const_data_ptr_cast(str.GetData());
|
43
|
+
return data[1] & ((1 << (8 - data[0])) - 1);
|
44
|
+
}
|
45
|
+
|
37
46
|
void Bit::Finalize(string_t &str) {
|
38
47
|
// bit strings require all padding bits to be set to 1
|
39
48
|
// this method sets all padding bits to 1
|
@@ -146,6 +155,48 @@ string Bit::ToBit(string_t str) {
|
|
146
155
|
return output_str.GetString();
|
147
156
|
}
|
148
157
|
|
158
|
+
void Bit::BlobToBit(string_t blob, string_t &output_str) {
|
159
|
+
auto data = const_data_ptr_cast(blob.GetData());
|
160
|
+
auto output = output_str.GetDataWriteable();
|
161
|
+
idx_t size = blob.GetSize();
|
162
|
+
|
163
|
+
*output = 0; // No padding
|
164
|
+
memcpy(output + 1, data, size);
|
165
|
+
}
|
166
|
+
|
167
|
+
string Bit::BlobToBit(string_t blob) {
|
168
|
+
auto buffer = make_unsafe_uniq_array<char>(blob.GetSize() + 1);
|
169
|
+
string_t output_str(buffer.get(), blob.GetSize() + 1);
|
170
|
+
Bit::BlobToBit(blob, output_str);
|
171
|
+
return output_str.GetString();
|
172
|
+
}
|
173
|
+
|
174
|
+
void Bit::BitToBlob(string_t bit, string_t &output_blob) {
|
175
|
+
D_ASSERT(bit.GetSize() == output_blob.GetSize() + 1);
|
176
|
+
|
177
|
+
auto data = const_data_ptr_cast(bit.GetData());
|
178
|
+
auto output = output_blob.GetDataWriteable();
|
179
|
+
idx_t size = output_blob.GetSize();
|
180
|
+
|
181
|
+
output[0] = GetFirstByte(bit);
|
182
|
+
if (size > 2) {
|
183
|
+
++output;
|
184
|
+
// First byte in bitstring contains amount of padded bits,
|
185
|
+
// second byte in bitstring is the padded byte,
|
186
|
+
// therefore the rest of the data starts at data + 2 (third byte)
|
187
|
+
memcpy(output, data + 2, size - 1);
|
188
|
+
}
|
189
|
+
}
|
190
|
+
|
191
|
+
string Bit::BitToBlob(string_t bit) {
|
192
|
+
D_ASSERT(bit.GetSize() > 1);
|
193
|
+
|
194
|
+
auto buffer = make_unsafe_uniq_array<char>(bit.GetSize() - 1);
|
195
|
+
string_t output_str(buffer.get(), bit.GetSize() - 1);
|
196
|
+
Bit::BitToBlob(bit, output_str);
|
197
|
+
return output_str.GetString();
|
198
|
+
}
|
199
|
+
|
149
200
|
// **** scalar functions ****
|
150
201
|
void Bit::BitString(const string_t &input, const idx_t &bit_length, string_t &result) {
|
151
202
|
char *res_buf = result.GetDataWriteable();
|
@@ -49,7 +49,7 @@ bool UUID::FromString(string str, hugeint_t &result) {
|
|
49
49
|
count++;
|
50
50
|
}
|
51
51
|
// Flip the first bit to make `order by uuid` same as `order by uuid::varchar`
|
52
|
-
result.upper ^= (
|
52
|
+
result.upper ^= (uint64_t(1) << 63);
|
53
53
|
return count == 32;
|
54
54
|
}
|
55
55
|
|
@@ -61,7 +61,7 @@ void UUID::ToString(hugeint_t input, char *buf) {
|
|
61
61
|
};
|
62
62
|
|
63
63
|
// Flip back before convert to string
|
64
|
-
int64_t upper = input.upper ^ (
|
64
|
+
int64_t upper = input.upper ^ (uint64_t(1) << 63);
|
65
65
|
idx_t pos = 0;
|
66
66
|
byte_to_hex(upper >> 56 & 0xFF, buf, pos);
|
67
67
|
byte_to_hex(upper >> 48 & 0xFF, buf, pos);
|
@@ -28,7 +28,7 @@ unique_ptr<FileHandle> VirtualFileSystem::OpenFile(const string &path, uint8_t f
|
|
28
28
|
}
|
29
29
|
}
|
30
30
|
// open the base file handle
|
31
|
-
auto file_handle = FindFileSystem(path)
|
31
|
+
auto file_handle = FindFileSystem(path).OpenFile(path, flags, lock, FileCompressionType::UNCOMPRESSED, opener);
|
32
32
|
if (file_handle->GetType() == FileType::FILE_TYPE_FIFO) {
|
33
33
|
file_handle = PipeFileSystem::OpenPipe(std::move(file_handle));
|
34
34
|
} else if (compression != FileCompressionType::UNCOMPRESSED) {
|
@@ -42,4 +42,141 @@ unique_ptr<FileHandle> VirtualFileSystem::OpenFile(const string &path, uint8_t f
|
|
42
42
|
return file_handle;
|
43
43
|
}
|
44
44
|
|
45
|
+
void VirtualFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) {
|
46
|
+
handle.file_system.Read(handle, buffer, nr_bytes, location);
|
47
|
+
}
|
48
|
+
|
49
|
+
void VirtualFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) {
|
50
|
+
handle.file_system.Write(handle, buffer, nr_bytes, location);
|
51
|
+
}
|
52
|
+
|
53
|
+
int64_t VirtualFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes) {
|
54
|
+
return handle.file_system.Read(handle, buffer, nr_bytes);
|
55
|
+
}
|
56
|
+
|
57
|
+
int64_t VirtualFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_bytes) {
|
58
|
+
return handle.file_system.Write(handle, buffer, nr_bytes);
|
59
|
+
}
|
60
|
+
|
61
|
+
int64_t VirtualFileSystem::GetFileSize(FileHandle &handle) {
|
62
|
+
return handle.file_system.GetFileSize(handle);
|
63
|
+
}
|
64
|
+
time_t VirtualFileSystem::GetLastModifiedTime(FileHandle &handle) {
|
65
|
+
return handle.file_system.GetLastModifiedTime(handle);
|
66
|
+
}
|
67
|
+
FileType VirtualFileSystem::GetFileType(FileHandle &handle) {
|
68
|
+
return handle.file_system.GetFileType(handle);
|
69
|
+
}
|
70
|
+
|
71
|
+
void VirtualFileSystem::Truncate(FileHandle &handle, int64_t new_size) {
|
72
|
+
handle.file_system.Truncate(handle, new_size);
|
73
|
+
}
|
74
|
+
|
75
|
+
void VirtualFileSystem::FileSync(FileHandle &handle) {
|
76
|
+
handle.file_system.FileSync(handle);
|
77
|
+
}
|
78
|
+
|
79
|
+
// need to look up correct fs for this
|
80
|
+
bool VirtualFileSystem::DirectoryExists(const string &directory) {
|
81
|
+
return FindFileSystem(directory).DirectoryExists(directory);
|
82
|
+
}
|
83
|
+
void VirtualFileSystem::CreateDirectory(const string &directory) {
|
84
|
+
FindFileSystem(directory).CreateDirectory(directory);
|
85
|
+
}
|
86
|
+
|
87
|
+
void VirtualFileSystem::RemoveDirectory(const string &directory) {
|
88
|
+
FindFileSystem(directory).RemoveDirectory(directory);
|
89
|
+
}
|
90
|
+
|
91
|
+
bool VirtualFileSystem::ListFiles(const string &directory, const std::function<void(const string &, bool)> &callback,
|
92
|
+
FileOpener *opener) {
|
93
|
+
return FindFileSystem(directory).ListFiles(directory, callback, opener);
|
94
|
+
}
|
95
|
+
|
96
|
+
void VirtualFileSystem::MoveFile(const string &source, const string &target) {
|
97
|
+
FindFileSystem(source).MoveFile(source, target);
|
98
|
+
}
|
99
|
+
|
100
|
+
bool VirtualFileSystem::FileExists(const string &filename) {
|
101
|
+
return FindFileSystem(filename).FileExists(filename);
|
102
|
+
}
|
103
|
+
|
104
|
+
bool VirtualFileSystem::IsPipe(const string &filename) {
|
105
|
+
return FindFileSystem(filename).IsPipe(filename);
|
106
|
+
}
|
107
|
+
void VirtualFileSystem::RemoveFile(const string &filename) {
|
108
|
+
FindFileSystem(filename).RemoveFile(filename);
|
109
|
+
}
|
110
|
+
|
111
|
+
vector<string> VirtualFileSystem::Glob(const string &path, FileOpener *opener) {
|
112
|
+
return FindFileSystem(path).Glob(path, opener);
|
113
|
+
}
|
114
|
+
|
115
|
+
void VirtualFileSystem::RegisterSubSystem(unique_ptr<FileSystem> fs) {
|
116
|
+
sub_systems.push_back(std::move(fs));
|
117
|
+
}
|
118
|
+
|
119
|
+
void VirtualFileSystem::UnregisterSubSystem(const string &name) {
|
120
|
+
for (auto sub_system = sub_systems.begin(); sub_system != sub_systems.end(); sub_system++) {
|
121
|
+
if (sub_system->get()->GetName() == name) {
|
122
|
+
sub_systems.erase(sub_system);
|
123
|
+
return;
|
124
|
+
}
|
125
|
+
}
|
126
|
+
throw InvalidInputException("Could not find filesystem with name %s", name);
|
127
|
+
}
|
128
|
+
|
129
|
+
void VirtualFileSystem::RegisterSubSystem(FileCompressionType compression_type, unique_ptr<FileSystem> fs) {
|
130
|
+
compressed_fs[compression_type] = std::move(fs);
|
131
|
+
}
|
132
|
+
|
133
|
+
vector<string> VirtualFileSystem::ListSubSystems() {
|
134
|
+
vector<string> names(sub_systems.size());
|
135
|
+
for (idx_t i = 0; i < sub_systems.size(); i++) {
|
136
|
+
names[i] = sub_systems[i]->GetName();
|
137
|
+
}
|
138
|
+
return names;
|
139
|
+
}
|
140
|
+
|
141
|
+
std::string VirtualFileSystem::GetName() const {
|
142
|
+
return "VirtualFileSystem";
|
143
|
+
}
|
144
|
+
|
145
|
+
void VirtualFileSystem::SetDisabledFileSystems(const vector<string> &names) {
|
146
|
+
unordered_set<string> new_disabled_file_systems;
|
147
|
+
for (auto &name : names) {
|
148
|
+
if (name.empty()) {
|
149
|
+
continue;
|
150
|
+
}
|
151
|
+
if (new_disabled_file_systems.find(name) != new_disabled_file_systems.end()) {
|
152
|
+
throw InvalidInputException("Duplicate disabled file system \"%s\"", name);
|
153
|
+
}
|
154
|
+
new_disabled_file_systems.insert(name);
|
155
|
+
}
|
156
|
+
for (auto &disabled_fs : disabled_file_systems) {
|
157
|
+
if (new_disabled_file_systems.find(disabled_fs) == new_disabled_file_systems.end()) {
|
158
|
+
throw InvalidInputException("File system \"%s\" has been disabled previously, it cannot be re-enabled",
|
159
|
+
disabled_fs);
|
160
|
+
}
|
161
|
+
}
|
162
|
+
disabled_file_systems = std::move(new_disabled_file_systems);
|
163
|
+
}
|
164
|
+
|
165
|
+
FileSystem &VirtualFileSystem::FindFileSystem(const string &path) {
|
166
|
+
auto &fs = FindFileSystemInternal(path);
|
167
|
+
if (!disabled_file_systems.empty() && disabled_file_systems.find(fs.GetName()) != disabled_file_systems.end()) {
|
168
|
+
throw PermissionException("File system %s has been disabled by configuration", fs.GetName());
|
169
|
+
}
|
170
|
+
return fs;
|
171
|
+
}
|
172
|
+
|
173
|
+
FileSystem &VirtualFileSystem::FindFileSystemInternal(const string &path) {
|
174
|
+
for (auto &sub_system : sub_systems) {
|
175
|
+
if (sub_system->CanHandleFile(path)) {
|
176
|
+
return *sub_system;
|
177
|
+
}
|
178
|
+
}
|
179
|
+
return *default_fs;
|
180
|
+
}
|
181
|
+
|
45
182
|
} // namespace duckdb
|
@@ -137,9 +137,9 @@ static StaticFunctionDefinition internal_functions[] = {
|
|
137
137
|
DUCKDB_SCALAR_FUNCTION(EnumRangeFun),
|
138
138
|
DUCKDB_SCALAR_FUNCTION(EnumRangeBoundaryFun),
|
139
139
|
DUCKDB_SCALAR_FUNCTION_SET(EpochFun),
|
140
|
-
DUCKDB_SCALAR_FUNCTION_SET(EpochNanosecondsFun),
|
141
|
-
DUCKDB_SCALAR_FUNCTION_SET(EpochMicrosecondsFun),
|
142
140
|
DUCKDB_SCALAR_FUNCTION_SET(EpochMsFun),
|
141
|
+
DUCKDB_SCALAR_FUNCTION_SET(EpochNsFun),
|
142
|
+
DUCKDB_SCALAR_FUNCTION_SET(EpochUsFun),
|
143
143
|
DUCKDB_SCALAR_FUNCTION_SET(EraFun),
|
144
144
|
DUCKDB_SCALAR_FUNCTION(ErrorFun),
|
145
145
|
DUCKDB_SCALAR_FUNCTION(EvenFun),
|
@@ -1591,7 +1591,7 @@ ScalarFunctionSet EpochFun::GetFunctions() {
|
|
1591
1591
|
return GetTimePartFunction<DatePart::EpochOperator>();
|
1592
1592
|
}
|
1593
1593
|
|
1594
|
-
ScalarFunctionSet
|
1594
|
+
ScalarFunctionSet EpochNsFun::GetFunctions() {
|
1595
1595
|
using OP = DatePart::EpochNanosecondsOperator;
|
1596
1596
|
auto operator_set = GetTimePartFunction<OP>();
|
1597
1597
|
|
@@ -1603,7 +1603,7 @@ ScalarFunctionSet EpochNanosecondsFun::GetFunctions() {
|
|
1603
1603
|
return operator_set;
|
1604
1604
|
}
|
1605
1605
|
|
1606
|
-
ScalarFunctionSet
|
1606
|
+
ScalarFunctionSet EpochUsFun::GetFunctions() {
|
1607
1607
|
using OP = DatePart::EpochMicrosecondsOperator;
|
1608
1608
|
auto operator_set = GetTimePartFunction<OP>();
|
1609
1609
|
|
@@ -1,7 +1,6 @@
|
|
1
1
|
#include "duckdb/core_functions/scalar/date_functions.hpp"
|
2
|
-
|
3
|
-
#include "duckdb/common/
|
4
|
-
#include "duckdb/common/types/timestamp.hpp"
|
2
|
+
|
3
|
+
#include "duckdb/common/operator/cast_operators.hpp"
|
5
4
|
#include "duckdb/common/vector_operations/vector_operations.hpp"
|
6
5
|
#include "duckdb/common/vector_operations/unary_executor.hpp"
|
7
6
|
|
@@ -9,20 +8,24 @@ namespace duckdb {
|
|
9
8
|
|
10
9
|
struct EpochSecOperator {
|
11
10
|
template <class INPUT_TYPE, class RESULT_TYPE>
|
12
|
-
static RESULT_TYPE Operation(INPUT_TYPE
|
13
|
-
|
11
|
+
static RESULT_TYPE Operation(INPUT_TYPE sec) {
|
12
|
+
int64_t result;
|
13
|
+
if (!TryCast::Operation(sec * Interval::MICROS_PER_SEC, result)) {
|
14
|
+
throw ConversionException("Could not convert epoch seconds to TIMESTAMP WITH TIME ZONE");
|
15
|
+
}
|
16
|
+
return timestamp_t(result);
|
14
17
|
}
|
15
18
|
};
|
16
19
|
|
17
20
|
static void EpochSecFunction(DataChunk &input, ExpressionState &state, Vector &result) {
|
18
21
|
D_ASSERT(input.ColumnCount() == 1);
|
19
22
|
|
20
|
-
UnaryExecutor::Execute<
|
23
|
+
UnaryExecutor::Execute<double, timestamp_t, EpochSecOperator>(input.data[0], result, input.size());
|
21
24
|
}
|
22
25
|
|
23
26
|
ScalarFunction ToTimestampFun::GetFunction() {
|
24
27
|
// to_timestamp is an alias from Postgres that converts the time in seconds to a timestamp
|
25
|
-
return ScalarFunction({LogicalType::
|
28
|
+
return ScalarFunction({LogicalType::DOUBLE}, LogicalType::TIMESTAMP_TZ, EpochSecFunction);
|
26
29
|
}
|
27
30
|
|
28
31
|
} // namespace duckdb
|
@@ -395,15 +395,15 @@ struct WindowColumnIterator {
|
|
395
395
|
return coll->GetCell<T>(pos + m);
|
396
396
|
}
|
397
397
|
|
398
|
-
friend inline iterator
|
398
|
+
friend inline iterator operator+(const iterator &a, difference_type n) {
|
399
399
|
return iterator(a.coll, a.pos + n);
|
400
400
|
}
|
401
401
|
|
402
|
-
friend inline iterator
|
402
|
+
friend inline iterator operator-(const iterator &a, difference_type n) {
|
403
403
|
return iterator(a.coll, a.pos - n);
|
404
404
|
}
|
405
405
|
|
406
|
-
friend inline iterator
|
406
|
+
friend inline iterator operator+(difference_type n, const iterator &a) {
|
407
407
|
return a + n;
|
408
408
|
}
|
409
409
|
friend inline difference_type operator-(const iterator &a, const iterator &b) {
|
@@ -669,7 +669,7 @@ public:
|
|
669
669
|
//! The read cursor
|
670
670
|
unique_ptr<PayloadScanner> scanner;
|
671
671
|
//! Pointer to the matches
|
672
|
-
const bool *found_match;
|
672
|
+
const bool *found_match = {};
|
673
673
|
};
|
674
674
|
|
675
675
|
AsOfLocalSourceState::AsOfLocalSourceState(AsOfGlobalSourceState &gsource, const PhysicalAsOfJoin &op)
|
@@ -167,8 +167,8 @@ PhysicalRangeJoin::PhysicalRangeJoin(LogicalOperator &op, PhysicalOperatorType t
|
|
167
167
|
// TODO: use stats to improve the choice?
|
168
168
|
// TODO: Prefer fixed length types?
|
169
169
|
if (conditions.size() > 1) {
|
170
|
-
|
171
|
-
|
170
|
+
vector<JoinCondition> conditions_p(conditions.size());
|
171
|
+
std::swap(conditions_p, conditions);
|
172
172
|
idx_t range_position = 0;
|
173
173
|
idx_t other_position = conditions_p.size();
|
174
174
|
for (idx_t i = 0; i < conditions_p.size(); ++i) {
|
@@ -13,7 +13,7 @@ CSVFileHandle::CSVFileHandle(FileSystem &fs, Allocator &allocator, unique_ptr<Fi
|
|
13
13
|
|
14
14
|
unique_ptr<FileHandle> CSVFileHandle::OpenFileHandle(FileSystem &fs, Allocator &allocator, const string &path,
|
15
15
|
FileCompressionType compression) {
|
16
|
-
auto file_handle = fs.OpenFile(path
|
16
|
+
auto file_handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_READ, FileLockType::NO_LOCK, compression);
|
17
17
|
if (file_handle->CanSeek()) {
|
18
18
|
file_handle->Reset();
|
19
19
|
}
|
@@ -41,7 +41,7 @@ PhysicalInsert::PhysicalInsert(vector<LogicalType> types_p, TableCatalogEntry &t
|
|
41
41
|
return;
|
42
42
|
}
|
43
43
|
|
44
|
-
D_ASSERT(set_expressions.size() == set_columns.size());
|
44
|
+
D_ASSERT(this->set_expressions.size() == this->set_columns.size());
|
45
45
|
|
46
46
|
// One or more columns are referenced from the existing table,
|
47
47
|
// we use the 'insert_types' to figure out which types these columns have
|