duckdb 0.8.2-dev1182.0 → 0.8.2-dev1435.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/binding.gyp +7 -7
  2. package/package.json +1 -1
  3. package/src/duckdb/extension/parquet/parquet_extension.cpp +1 -1
  4. package/src/duckdb/extension/parquet/parquet_reader.cpp +1 -1
  5. package/src/duckdb/src/common/arrow/arrow_appender.cpp +0 -1
  6. package/src/duckdb/src/common/exception.cpp +2 -2
  7. package/src/duckdb/src/common/file_system.cpp +4 -0
  8. package/src/duckdb/src/common/hive_partitioning.cpp +10 -6
  9. package/src/duckdb/src/common/local_file_system.cpp +1 -1
  10. package/src/duckdb/src/common/multi_file_reader.cpp +3 -2
  11. package/src/duckdb/src/common/operator/cast_operators.cpp +35 -1
  12. package/src/duckdb/src/common/re2_regex.cpp +1 -1
  13. package/src/duckdb/src/common/types/bit.cpp +51 -0
  14. package/src/duckdb/src/common/types/uuid.cpp +2 -2
  15. package/src/duckdb/src/common/virtual_file_system.cpp +138 -1
  16. package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp +2 -0
  17. package/src/duckdb/src/core_functions/function_list.cpp +2 -2
  18. package/src/duckdb/src/core_functions/scalar/date/date_part.cpp +2 -2
  19. package/src/duckdb/src/core_functions/scalar/date/epoch.cpp +10 -7
  20. package/src/duckdb/src/execution/expression_executor.cpp +1 -1
  21. package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +3 -3
  22. package/src/duckdb/src/execution/operator/join/physical_asof_join.cpp +1 -1
  23. package/src/duckdb/src/execution/operator/join/physical_range_join.cpp +2 -2
  24. package/src/duckdb/src/execution/operator/persistent/csv_file_handle.cpp +1 -1
  25. package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +1 -1
  26. package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp +7 -2
  27. package/src/duckdb/src/execution/physical_plan/plan_get.cpp +2 -2
  28. package/src/duckdb/src/execution/window_segment_tree.cpp +1 -1
  29. package/src/duckdb/src/function/cast/bit_cast.cpp +34 -2
  30. package/src/duckdb/src/function/cast/blob_cast.cpp +3 -0
  31. package/src/duckdb/src/function/cast/numeric_casts.cpp +2 -0
  32. package/src/duckdb/src/function/function.cpp +1 -1
  33. package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
  34. package/src/duckdb/src/include/duckdb/common/extra_operator_info.hpp +27 -0
  35. package/src/duckdb/src/include/duckdb/common/file_system.hpp +2 -0
  36. package/src/duckdb/src/include/duckdb/common/hive_partitioning.hpp +1 -1
  37. package/src/duckdb/src/include/duckdb/common/operator/cast_operators.hpp +43 -3
  38. package/src/duckdb/src/include/duckdb/common/operator/numeric_cast.hpp +10 -0
  39. package/src/duckdb/src/include/duckdb/common/types/bit.hpp +81 -0
  40. package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp +38 -97
  41. package/src/duckdb/src/include/duckdb/core_functions/scalar/date_functions.hpp +11 -11
  42. package/src/duckdb/src/include/duckdb/execution/operator/persistent/base_csv_reader.hpp +1 -1
  43. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp +5 -1
  44. package/src/duckdb/src/include/duckdb/main/connection.hpp +1 -2
  45. package/src/duckdb/src/include/duckdb/main/relation/cross_product_relation.hpp +4 -1
  46. package/src/duckdb/src/include/duckdb/main/relation/join_relation.hpp +5 -2
  47. package/src/duckdb/src/include/duckdb/main/relation.hpp +4 -2
  48. package/src/duckdb/src/include/duckdb/main/settings.hpp +9 -0
  49. package/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp +1 -1
  50. package/src/duckdb/src/include/duckdb/planner/constraints/bound_unique_constraint.hpp +3 -3
  51. package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp +4 -0
  52. package/src/duckdb/src/main/capi/duckdb_value-c.cpp +1 -1
  53. package/src/duckdb/src/main/config.cpp +1 -0
  54. package/src/duckdb/src/main/database.cpp +1 -1
  55. package/src/duckdb/src/main/extension/extension_install.cpp +6 -0
  56. package/src/duckdb/src/main/extension/extension_load.cpp +10 -1
  57. package/src/duckdb/src/main/relation/cross_product_relation.cpp +4 -3
  58. package/src/duckdb/src/main/relation/join_relation.cpp +5 -5
  59. package/src/duckdb/src/main/relation.cpp +6 -5
  60. package/src/duckdb/src/main/settings/settings.cpp +24 -0
  61. package/src/duckdb/src/parser/parser.cpp +8 -2
  62. package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +5 -4
  63. package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp +1 -1
  64. package/src/duckdb/src/planner/operator/logical_get.cpp +9 -4
  65. package/src/duckdb/src/storage/compression/rle.cpp +0 -1
  66. package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +6 -6
  67. package/src/statement.cpp +2 -1
package/binding.gyp CHANGED
@@ -239,18 +239,18 @@
239
239
  "src/duckdb/third_party/zstd/compress/zstd_lazy.cpp",
240
240
  "src/duckdb/third_party/zstd/compress/zstd_ldm.cpp",
241
241
  "src/duckdb/third_party/zstd/compress/zstd_opt.cpp",
242
- "src/duckdb/extension/icu/./icu-datepart.cpp",
243
- "src/duckdb/extension/icu/./icu-timezone.cpp",
244
- "src/duckdb/extension/icu/./icu-makedate.cpp",
245
- "src/duckdb/extension/icu/./icu-table-range.cpp",
246
- "src/duckdb/extension/icu/./icu-datefunc.cpp",
247
242
  "src/duckdb/extension/icu/./icu-list-range.cpp",
248
- "src/duckdb/extension/icu/./icu-dateadd.cpp",
243
+ "src/duckdb/extension/icu/./icu-table-range.cpp",
249
244
  "src/duckdb/extension/icu/./icu-datesub.cpp",
245
+ "src/duckdb/extension/icu/./icu-dateadd.cpp",
250
246
  "src/duckdb/extension/icu/./icu-timebucket.cpp",
251
- "src/duckdb/extension/icu/./icu-strptime.cpp",
247
+ "src/duckdb/extension/icu/./icu-makedate.cpp",
248
+ "src/duckdb/extension/icu/./icu-timezone.cpp",
249
+ "src/duckdb/extension/icu/./icu-datefunc.cpp",
250
+ "src/duckdb/extension/icu/./icu-datepart.cpp",
252
251
  "src/duckdb/extension/icu/./icu_extension.cpp",
253
252
  "src/duckdb/extension/icu/./icu-datetrunc.cpp",
253
+ "src/duckdb/extension/icu/./icu-strptime.cpp",
254
254
  "src/duckdb/ub_extension_icu_third_party_icu_common.cpp",
255
255
  "src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp",
256
256
  "src/duckdb/extension/icu/third_party/icu/stubdata/stubdata.cpp",
package/package.json CHANGED
@@ -2,7 +2,7 @@
2
2
  "name": "duckdb",
3
3
  "main": "./lib/duckdb.js",
4
4
  "types": "./lib/duckdb.d.ts",
5
- "version": "0.8.2-dev1182.0",
5
+ "version": "0.8.2-dev1435.0",
6
6
  "description": "DuckDB node.js API",
7
7
  "gypfile": true,
8
8
  "dependencies": {
@@ -175,7 +175,6 @@ public:
175
175
  table_function.serialize = ParquetScanSerialize;
176
176
  table_function.deserialize = ParquetScanDeserialize;
177
177
  table_function.get_batch_info = ParquetGetBatchInfo;
178
-
179
178
  table_function.projection_pushdown = true;
180
179
  table_function.filter_pushdown = true;
181
180
  table_function.filter_prune = true;
@@ -520,6 +519,7 @@ public:
520
519
  static void ParquetComplexFilterPushdown(ClientContext &context, LogicalGet &get, FunctionData *bind_data_p,
521
520
  vector<unique_ptr<Expression>> &filters) {
522
521
  auto &data = bind_data_p->Cast<ParquetReadBindData>();
522
+
523
523
  auto reset_reader = MultiFileReader::ComplexFilterPushdown(context, data.files,
524
524
  data.parquet_options.file_options, get, filters);
525
525
  if (reset_reader) {
@@ -76,7 +76,7 @@ static shared_ptr<ParquetFileMetadataCache> LoadMetadata(Allocator &allocator, F
76
76
  }
77
77
  // read four-byte footer length from just before the end magic bytes
78
78
  auto footer_len = *reinterpret_cast<uint32_t *>(buf.ptr);
79
- if (footer_len <= 0 || file_size < 12 + footer_len) {
79
+ if (footer_len == 0 || file_size < 12 + footer_len) {
80
80
  throw InvalidInputException("Footer length error in file '%s'", file_handle.path);
81
81
  }
82
82
  auto metadata_pos = file_size - (footer_len + 8);
@@ -786,7 +786,6 @@ ArrowArray ArrowAppender::Finalize() {
786
786
 
787
787
  // Configure root array
788
788
  result.length = row_count;
789
- result.n_children = types.size();
790
789
  result.n_buffers = 1;
791
790
  result.buffers = root_holder->buffers.data(); // there is no actual buffer there since we don't have NULLs
792
791
  result.offset = 0;
@@ -62,11 +62,11 @@ string Exception::ConstructMessageRecursive(const string &msg, std::vector<Excep
62
62
  #ifdef DEBUG
63
63
  // Verify that we have the required amount of values for the message
64
64
  idx_t parameter_count = 0;
65
- for (idx_t i = 0; i < msg.size(); i++) {
65
+ for (idx_t i = 0; i + 1 < msg.size(); i++) {
66
66
  if (msg[i] != '%') {
67
67
  continue;
68
68
  }
69
- if (i < msg.size() && msg[i + 1] == '%') {
69
+ if (msg[i + 1] == '%') {
70
70
  i++;
71
71
  continue;
72
72
  }
@@ -385,6 +385,10 @@ void FileSystem::UnregisterSubSystem(const string &name) {
385
385
  throw NotImplementedException("%s: Can't unregister a sub system on a non-virtual file system", GetName());
386
386
  }
387
387
 
388
+ void FileSystem::SetDisabledFileSystems(const vector<string> &names) {
389
+ throw NotImplementedException("%s: Can't disable file systems on a non-virtual file system", GetName());
390
+ }
391
+
388
392
  vector<string> FileSystem::ListSubSystems() {
389
393
  throw NotImplementedException("%s: Can't list sub systems on a non-virtual file system", GetName());
390
394
  }
@@ -6,6 +6,7 @@
6
6
  #include "duckdb/planner/expression/bound_constant_expression.hpp"
7
7
  #include "duckdb/planner/expression/bound_reference_expression.hpp"
8
8
  #include "duckdb/planner/expression_iterator.hpp"
9
+ #include "duckdb/planner/operator/logical_get.hpp"
9
10
  #include "duckdb/planner/table_filter.hpp"
10
11
  #include "re2/re2.h"
11
12
 
@@ -86,12 +87,15 @@ std::map<string, string> HivePartitioning::Parse(const string &filename) {
86
87
  // currently, only expressions that cannot be evaluated during pushdown are removed.
87
88
  void HivePartitioning::ApplyFiltersToFileList(ClientContext &context, vector<string> &files,
88
89
  vector<unique_ptr<Expression>> &filters,
89
- unordered_map<string, column_t> &column_map, idx_t table_index,
90
+ unordered_map<string, column_t> &column_map, LogicalGet &get,
90
91
  bool hive_enabled, bool filename_enabled) {
92
+
91
93
  vector<string> pruned_files;
92
94
  vector<bool> have_preserved_filter(filters.size(), false);
93
95
  vector<unique_ptr<Expression>> pruned_filters;
96
+ unordered_set<idx_t> filters_applied_to_files;
94
97
  duckdb_re2::RE2 regex(REGEX_STRING);
98
+ auto table_index = get.table_index;
95
99
 
96
100
  if ((!filename_enabled && !hive_enabled) || filters.empty()) {
97
101
  return;
@@ -121,11 +125,11 @@ void HivePartitioning::ApplyFiltersToFileList(ClientContext &context, vector<str
121
125
  } else if (!result_value.GetValue<bool>()) {
122
126
  // filter evaluates to false
123
127
  should_prune_file = true;
124
- }
125
-
126
- // Use filter combiner to determine that this filter makes
127
- if (!should_prune_file && combiner.AddFilter(std::move(filter_copy)) == FilterResult::UNSATISFIABLE) {
128
- should_prune_file = true;
128
+ // convert the filter to a table filter.
129
+ if (filters_applied_to_files.find(j) == filters_applied_to_files.end()) {
130
+ get.extra_info.file_filters += filter->ToString();
131
+ filters_applied_to_files.insert(j);
132
+ }
129
133
  }
130
134
  }
131
135
 
@@ -375,7 +375,7 @@ int RemoveDirectoryRecursive(const char *path) {
375
375
  continue;
376
376
  }
377
377
  len = path_len + (idx_t)strlen(p->d_name) + 2;
378
- buf = new char[len];
378
+ buf = new (std::nothrow) char[len];
379
379
  if (buf) {
380
380
  struct stat statbuf;
381
381
  snprintf(buf, len, "%s/%s", path, p->d_name);
@@ -106,8 +106,9 @@ bool MultiFileReader::ComplexFilterPushdown(ClientContext &context, vector<strin
106
106
  }
107
107
 
108
108
  auto start_files = files.size();
109
- HivePartitioning::ApplyFiltersToFileList(context, files, filters, column_map, get.table_index,
110
- options.hive_partitioning, options.filename);
109
+ HivePartitioning::ApplyFiltersToFileList(context, files, filters, column_map, get, options.hive_partitioning,
110
+ options.filename);
111
+
111
112
  if (files.size() != start_files) {
112
113
  // we have pruned files
113
114
  return true;
@@ -1,4 +1,5 @@
1
1
  #include "duckdb/common/operator/cast_operators.hpp"
2
+ #include "duckdb/common/hugeint.hpp"
2
3
  #include "duckdb/common/operator/string_cast.hpp"
3
4
  #include "duckdb/common/operator/numeric_cast.hpp"
4
5
  #include "duckdb/common/operator/decimal_cast_operators.hpp"
@@ -1425,11 +1426,20 @@ string_t CastFromBlob::Operation(string_t input, Vector &vector) {
1425
1426
  return result;
1426
1427
  }
1427
1428
 
1429
+ template <>
1430
+ string_t CastFromBlobToBit::Operation(string_t input, Vector &vector) {
1431
+ idx_t result_size = input.GetSize() + 1;
1432
+ if (result_size <= 1) {
1433
+ throw ConversionException("Cannot cast empty BLOB to BIT");
1434
+ }
1435
+ return StringVector::AddStringOrBlob(vector, Bit::BlobToBit(input));
1436
+ }
1437
+
1428
1438
  //===--------------------------------------------------------------------===//
1429
1439
  // Cast From Bit
1430
1440
  //===--------------------------------------------------------------------===//
1431
1441
  template <>
1432
- string_t CastFromBit::Operation(string_t input, Vector &vector) {
1442
+ string_t CastFromBitToString::Operation(string_t input, Vector &vector) {
1433
1443
 
1434
1444
  idx_t result_size = Bit::BitLength(input);
1435
1445
  string_t result = StringVector::EmptyString(vector, result_size);
@@ -1482,6 +1492,30 @@ bool TryCastToBit::Operation(string_t input, string_t &result, Vector &result_ve
1482
1492
  return true;
1483
1493
  }
1484
1494
 
1495
+ template <>
1496
+ bool CastFromBitToNumeric::Operation(string_t input, bool &result, bool strict) {
1497
+ D_ASSERT(input.GetSize() > 1);
1498
+
1499
+ uint8_t value;
1500
+ bool success = CastFromBitToNumeric::Operation(input, value, strict);
1501
+ result = (value > 0);
1502
+ return (success);
1503
+ }
1504
+
1505
+ template <>
1506
+ bool CastFromBitToNumeric::Operation(string_t input, hugeint_t &result, bool strict) {
1507
+ D_ASSERT(input.GetSize() > 1);
1508
+
1509
+ if (input.GetSize() - 1 > sizeof(hugeint_t)) {
1510
+ throw ConversionException("Bitstring doesn't fit inside of %s", GetTypeId<hugeint_t>());
1511
+ }
1512
+ Bit::BitToNumeric(input, result);
1513
+ if (result < NumericLimits<hugeint_t>::Minimum()) {
1514
+ throw ConversionException("Minimum limit for HUGEINT is %s", NumericLimits<hugeint_t>::Minimum().ToString());
1515
+ }
1516
+ return (true);
1517
+ }
1518
+
1485
1519
  //===--------------------------------------------------------------------===//
1486
1520
  // Cast From UUID
1487
1521
  //===--------------------------------------------------------------------===//
@@ -54,7 +54,7 @@ duckdb::vector<Match> RegexFindAll(const std::string &input, const Regex &regex)
54
54
  Match match;
55
55
  while (RegexSearchInternal(input.c_str(), match, regex, RE2::UNANCHORED, position, input.size())) {
56
56
  position += match.position(0) + match.length(0);
57
- matches.emplace_back(std::move(match));
57
+ matches.emplace_back(match);
58
58
  }
59
59
  return matches;
60
60
  }
@@ -1,4 +1,6 @@
1
+ #include "duckdb/common/assert.hpp"
1
2
  #include "duckdb/common/operator/cast_operators.hpp"
3
+ #include "duckdb/common/typedefs.hpp"
2
4
  #include "duckdb/common/types/bit.hpp"
3
5
  #include "duckdb/common/types/string_type.hpp"
4
6
 
@@ -34,6 +36,13 @@ static inline idx_t GetBitSize(const string_t &str) {
34
36
  return str_len;
35
37
  }
36
38
 
39
+ uint8_t Bit::GetFirstByte(const string_t &str) {
40
+ D_ASSERT(str.GetSize() > 1);
41
+
42
+ auto data = const_data_ptr_cast(str.GetData());
43
+ return data[1] & ((1 << (8 - data[0])) - 1);
44
+ }
45
+
37
46
  void Bit::Finalize(string_t &str) {
38
47
  // bit strings require all padding bits to be set to 1
39
48
  // this method sets all padding bits to 1
@@ -146,6 +155,48 @@ string Bit::ToBit(string_t str) {
146
155
  return output_str.GetString();
147
156
  }
148
157
 
158
+ void Bit::BlobToBit(string_t blob, string_t &output_str) {
159
+ auto data = const_data_ptr_cast(blob.GetData());
160
+ auto output = output_str.GetDataWriteable();
161
+ idx_t size = blob.GetSize();
162
+
163
+ *output = 0; // No padding
164
+ memcpy(output + 1, data, size);
165
+ }
166
+
167
+ string Bit::BlobToBit(string_t blob) {
168
+ auto buffer = make_unsafe_uniq_array<char>(blob.GetSize() + 1);
169
+ string_t output_str(buffer.get(), blob.GetSize() + 1);
170
+ Bit::BlobToBit(blob, output_str);
171
+ return output_str.GetString();
172
+ }
173
+
174
+ void Bit::BitToBlob(string_t bit, string_t &output_blob) {
175
+ D_ASSERT(bit.GetSize() == output_blob.GetSize() + 1);
176
+
177
+ auto data = const_data_ptr_cast(bit.GetData());
178
+ auto output = output_blob.GetDataWriteable();
179
+ idx_t size = output_blob.GetSize();
180
+
181
+ output[0] = GetFirstByte(bit);
182
+ if (size > 2) {
183
+ ++output;
184
+ // First byte in bitstring contains amount of padded bits,
185
+ // second byte in bitstring is the padded byte,
186
+ // therefore the rest of the data starts at data + 2 (third byte)
187
+ memcpy(output, data + 2, size - 1);
188
+ }
189
+ }
190
+
191
+ string Bit::BitToBlob(string_t bit) {
192
+ D_ASSERT(bit.GetSize() > 1);
193
+
194
+ auto buffer = make_unsafe_uniq_array<char>(bit.GetSize() - 1);
195
+ string_t output_str(buffer.get(), bit.GetSize() - 1);
196
+ Bit::BitToBlob(bit, output_str);
197
+ return output_str.GetString();
198
+ }
199
+
149
200
  // **** scalar functions ****
150
201
  void Bit::BitString(const string_t &input, const idx_t &bit_length, string_t &result) {
151
202
  char *res_buf = result.GetDataWriteable();
@@ -49,7 +49,7 @@ bool UUID::FromString(string str, hugeint_t &result) {
49
49
  count++;
50
50
  }
51
51
  // Flip the first bit to make `order by uuid` same as `order by uuid::varchar`
52
- result.upper ^= (int64_t(1) << 63);
52
+ result.upper ^= (uint64_t(1) << 63);
53
53
  return count == 32;
54
54
  }
55
55
 
@@ -61,7 +61,7 @@ void UUID::ToString(hugeint_t input, char *buf) {
61
61
  };
62
62
 
63
63
  // Flip back before convert to string
64
- int64_t upper = input.upper ^ (int64_t(1) << 63);
64
+ int64_t upper = input.upper ^ (uint64_t(1) << 63);
65
65
  idx_t pos = 0;
66
66
  byte_to_hex(upper >> 56 & 0xFF, buf, pos);
67
67
  byte_to_hex(upper >> 48 & 0xFF, buf, pos);
@@ -28,7 +28,7 @@ unique_ptr<FileHandle> VirtualFileSystem::OpenFile(const string &path, uint8_t f
28
28
  }
29
29
  }
30
30
  // open the base file handle
31
- auto file_handle = FindFileSystem(path)->OpenFile(path, flags, lock, FileCompressionType::UNCOMPRESSED, opener);
31
+ auto file_handle = FindFileSystem(path).OpenFile(path, flags, lock, FileCompressionType::UNCOMPRESSED, opener);
32
32
  if (file_handle->GetType() == FileType::FILE_TYPE_FIFO) {
33
33
  file_handle = PipeFileSystem::OpenPipe(std::move(file_handle));
34
34
  } else if (compression != FileCompressionType::UNCOMPRESSED) {
@@ -42,4 +42,141 @@ unique_ptr<FileHandle> VirtualFileSystem::OpenFile(const string &path, uint8_t f
42
42
  return file_handle;
43
43
  }
44
44
 
45
+ void VirtualFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) {
46
+ handle.file_system.Read(handle, buffer, nr_bytes, location);
47
+ }
48
+
49
+ void VirtualFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) {
50
+ handle.file_system.Write(handle, buffer, nr_bytes, location);
51
+ }
52
+
53
+ int64_t VirtualFileSystem::Read(FileHandle &handle, void *buffer, int64_t nr_bytes) {
54
+ return handle.file_system.Read(handle, buffer, nr_bytes);
55
+ }
56
+
57
+ int64_t VirtualFileSystem::Write(FileHandle &handle, void *buffer, int64_t nr_bytes) {
58
+ return handle.file_system.Write(handle, buffer, nr_bytes);
59
+ }
60
+
61
+ int64_t VirtualFileSystem::GetFileSize(FileHandle &handle) {
62
+ return handle.file_system.GetFileSize(handle);
63
+ }
64
+ time_t VirtualFileSystem::GetLastModifiedTime(FileHandle &handle) {
65
+ return handle.file_system.GetLastModifiedTime(handle);
66
+ }
67
+ FileType VirtualFileSystem::GetFileType(FileHandle &handle) {
68
+ return handle.file_system.GetFileType(handle);
69
+ }
70
+
71
+ void VirtualFileSystem::Truncate(FileHandle &handle, int64_t new_size) {
72
+ handle.file_system.Truncate(handle, new_size);
73
+ }
74
+
75
+ void VirtualFileSystem::FileSync(FileHandle &handle) {
76
+ handle.file_system.FileSync(handle);
77
+ }
78
+
79
+ // need to look up correct fs for this
80
+ bool VirtualFileSystem::DirectoryExists(const string &directory) {
81
+ return FindFileSystem(directory).DirectoryExists(directory);
82
+ }
83
+ void VirtualFileSystem::CreateDirectory(const string &directory) {
84
+ FindFileSystem(directory).CreateDirectory(directory);
85
+ }
86
+
87
+ void VirtualFileSystem::RemoveDirectory(const string &directory) {
88
+ FindFileSystem(directory).RemoveDirectory(directory);
89
+ }
90
+
91
+ bool VirtualFileSystem::ListFiles(const string &directory, const std::function<void(const string &, bool)> &callback,
92
+ FileOpener *opener) {
93
+ return FindFileSystem(directory).ListFiles(directory, callback, opener);
94
+ }
95
+
96
+ void VirtualFileSystem::MoveFile(const string &source, const string &target) {
97
+ FindFileSystem(source).MoveFile(source, target);
98
+ }
99
+
100
+ bool VirtualFileSystem::FileExists(const string &filename) {
101
+ return FindFileSystem(filename).FileExists(filename);
102
+ }
103
+
104
+ bool VirtualFileSystem::IsPipe(const string &filename) {
105
+ return FindFileSystem(filename).IsPipe(filename);
106
+ }
107
+ void VirtualFileSystem::RemoveFile(const string &filename) {
108
+ FindFileSystem(filename).RemoveFile(filename);
109
+ }
110
+
111
+ vector<string> VirtualFileSystem::Glob(const string &path, FileOpener *opener) {
112
+ return FindFileSystem(path).Glob(path, opener);
113
+ }
114
+
115
+ void VirtualFileSystem::RegisterSubSystem(unique_ptr<FileSystem> fs) {
116
+ sub_systems.push_back(std::move(fs));
117
+ }
118
+
119
+ void VirtualFileSystem::UnregisterSubSystem(const string &name) {
120
+ for (auto sub_system = sub_systems.begin(); sub_system != sub_systems.end(); sub_system++) {
121
+ if (sub_system->get()->GetName() == name) {
122
+ sub_systems.erase(sub_system);
123
+ return;
124
+ }
125
+ }
126
+ throw InvalidInputException("Could not find filesystem with name %s", name);
127
+ }
128
+
129
+ void VirtualFileSystem::RegisterSubSystem(FileCompressionType compression_type, unique_ptr<FileSystem> fs) {
130
+ compressed_fs[compression_type] = std::move(fs);
131
+ }
132
+
133
+ vector<string> VirtualFileSystem::ListSubSystems() {
134
+ vector<string> names(sub_systems.size());
135
+ for (idx_t i = 0; i < sub_systems.size(); i++) {
136
+ names[i] = sub_systems[i]->GetName();
137
+ }
138
+ return names;
139
+ }
140
+
141
+ std::string VirtualFileSystem::GetName() const {
142
+ return "VirtualFileSystem";
143
+ }
144
+
145
+ void VirtualFileSystem::SetDisabledFileSystems(const vector<string> &names) {
146
+ unordered_set<string> new_disabled_file_systems;
147
+ for (auto &name : names) {
148
+ if (name.empty()) {
149
+ continue;
150
+ }
151
+ if (new_disabled_file_systems.find(name) != new_disabled_file_systems.end()) {
152
+ throw InvalidInputException("Duplicate disabled file system \"%s\"", name);
153
+ }
154
+ new_disabled_file_systems.insert(name);
155
+ }
156
+ for (auto &disabled_fs : disabled_file_systems) {
157
+ if (new_disabled_file_systems.find(disabled_fs) == new_disabled_file_systems.end()) {
158
+ throw InvalidInputException("File system \"%s\" has been disabled previously, it cannot be re-enabled",
159
+ disabled_fs);
160
+ }
161
+ }
162
+ disabled_file_systems = std::move(new_disabled_file_systems);
163
+ }
164
+
165
+ FileSystem &VirtualFileSystem::FindFileSystem(const string &path) {
166
+ auto &fs = FindFileSystemInternal(path);
167
+ if (!disabled_file_systems.empty() && disabled_file_systems.find(fs.GetName()) != disabled_file_systems.end()) {
168
+ throw PermissionException("File system %s has been disabled by configuration", fs.GetName());
169
+ }
170
+ return fs;
171
+ }
172
+
173
+ FileSystem &VirtualFileSystem::FindFileSystemInternal(const string &path) {
174
+ for (auto &sub_system : sub_systems) {
175
+ if (sub_system->CanHandleFile(path)) {
176
+ return *sub_system;
177
+ }
178
+ }
179
+ return *default_fs;
180
+ }
181
+
45
182
  } // namespace duckdb
@@ -21,8 +21,10 @@ struct ReservoirQuantileState {
21
21
  if (new_len <= len) {
22
22
  return;
23
23
  }
24
+ T *old_v = v;
24
25
  v = (T *)realloc(v, new_len * sizeof(T));
25
26
  if (!v) {
27
+ free(old_v);
26
28
  throw InternalException("Memory allocation failure");
27
29
  }
28
30
  len = new_len;
@@ -137,9 +137,9 @@ static StaticFunctionDefinition internal_functions[] = {
137
137
  DUCKDB_SCALAR_FUNCTION(EnumRangeFun),
138
138
  DUCKDB_SCALAR_FUNCTION(EnumRangeBoundaryFun),
139
139
  DUCKDB_SCALAR_FUNCTION_SET(EpochFun),
140
- DUCKDB_SCALAR_FUNCTION_SET(EpochNanosecondsFun),
141
- DUCKDB_SCALAR_FUNCTION_SET(EpochMicrosecondsFun),
142
140
  DUCKDB_SCALAR_FUNCTION_SET(EpochMsFun),
141
+ DUCKDB_SCALAR_FUNCTION_SET(EpochNsFun),
142
+ DUCKDB_SCALAR_FUNCTION_SET(EpochUsFun),
143
143
  DUCKDB_SCALAR_FUNCTION_SET(EraFun),
144
144
  DUCKDB_SCALAR_FUNCTION(ErrorFun),
145
145
  DUCKDB_SCALAR_FUNCTION(EvenFun),
@@ -1591,7 +1591,7 @@ ScalarFunctionSet EpochFun::GetFunctions() {
1591
1591
  return GetTimePartFunction<DatePart::EpochOperator>();
1592
1592
  }
1593
1593
 
1594
- ScalarFunctionSet EpochNanosecondsFun::GetFunctions() {
1594
+ ScalarFunctionSet EpochNsFun::GetFunctions() {
1595
1595
  using OP = DatePart::EpochNanosecondsOperator;
1596
1596
  auto operator_set = GetTimePartFunction<OP>();
1597
1597
 
@@ -1603,7 +1603,7 @@ ScalarFunctionSet EpochNanosecondsFun::GetFunctions() {
1603
1603
  return operator_set;
1604
1604
  }
1605
1605
 
1606
- ScalarFunctionSet EpochMicrosecondsFun::GetFunctions() {
1606
+ ScalarFunctionSet EpochUsFun::GetFunctions() {
1607
1607
  using OP = DatePart::EpochMicrosecondsOperator;
1608
1608
  auto operator_set = GetTimePartFunction<OP>();
1609
1609
 
@@ -1,7 +1,6 @@
1
1
  #include "duckdb/core_functions/scalar/date_functions.hpp"
2
- #include "duckdb/common/types/time.hpp"
3
- #include "duckdb/common/types/date.hpp"
4
- #include "duckdb/common/types/timestamp.hpp"
2
+
3
+ #include "duckdb/common/operator/cast_operators.hpp"
5
4
  #include "duckdb/common/vector_operations/vector_operations.hpp"
6
5
  #include "duckdb/common/vector_operations/unary_executor.hpp"
7
6
 
@@ -9,20 +8,24 @@ namespace duckdb {
9
8
 
10
9
  struct EpochSecOperator {
11
10
  template <class INPUT_TYPE, class RESULT_TYPE>
12
- static RESULT_TYPE Operation(INPUT_TYPE input) {
13
- return Timestamp::FromEpochSeconds(input);
11
+ static RESULT_TYPE Operation(INPUT_TYPE sec) {
12
+ int64_t result;
13
+ if (!TryCast::Operation(sec * Interval::MICROS_PER_SEC, result)) {
14
+ throw ConversionException("Could not convert epoch seconds to TIMESTAMP WITH TIME ZONE");
15
+ }
16
+ return timestamp_t(result);
14
17
  }
15
18
  };
16
19
 
17
20
  static void EpochSecFunction(DataChunk &input, ExpressionState &state, Vector &result) {
18
21
  D_ASSERT(input.ColumnCount() == 1);
19
22
 
20
- UnaryExecutor::Execute<int64_t, timestamp_t, EpochSecOperator>(input.data[0], result, input.size());
23
+ UnaryExecutor::Execute<double, timestamp_t, EpochSecOperator>(input.data[0], result, input.size());
21
24
  }
22
25
 
23
26
  ScalarFunction ToTimestampFun::GetFunction() {
24
27
  // to_timestamp is an alias from Postgres that converts the time in seconds to a timestamp
25
- return ScalarFunction({LogicalType::BIGINT}, LogicalType::TIMESTAMP, EpochSecFunction);
28
+ return ScalarFunction({LogicalType::DOUBLE}, LogicalType::TIMESTAMP_TZ, EpochSecFunction);
26
29
  }
27
30
 
28
31
  } // namespace duckdb
@@ -126,7 +126,7 @@ bool ExpressionExecutor::TryEvaluateScalar(ClientContext &context, const Express
126
126
  result = EvaluateScalar(context, expr);
127
127
  return true;
128
128
  } catch (InternalException &ex) {
129
- throw ex;
129
+ throw;
130
130
  } catch (...) {
131
131
  return false;
132
132
  }
@@ -395,15 +395,15 @@ struct WindowColumnIterator {
395
395
  return coll->GetCell<T>(pos + m);
396
396
  }
397
397
 
398
- friend inline iterator &operator+(const iterator &a, difference_type n) {
398
+ friend inline iterator operator+(const iterator &a, difference_type n) {
399
399
  return iterator(a.coll, a.pos + n);
400
400
  }
401
401
 
402
- friend inline iterator &operator-(const iterator &a, difference_type n) {
402
+ friend inline iterator operator-(const iterator &a, difference_type n) {
403
403
  return iterator(a.coll, a.pos - n);
404
404
  }
405
405
 
406
- friend inline iterator &operator+(difference_type n, const iterator &a) {
406
+ friend inline iterator operator+(difference_type n, const iterator &a) {
407
407
  return a + n;
408
408
  }
409
409
  friend inline difference_type operator-(const iterator &a, const iterator &b) {
@@ -669,7 +669,7 @@ public:
669
669
  //! The read cursor
670
670
  unique_ptr<PayloadScanner> scanner;
671
671
  //! Pointer to the matches
672
- const bool *found_match;
672
+ const bool *found_match = {};
673
673
  };
674
674
 
675
675
  AsOfLocalSourceState::AsOfLocalSourceState(AsOfGlobalSourceState &gsource, const PhysicalAsOfJoin &op)
@@ -167,8 +167,8 @@ PhysicalRangeJoin::PhysicalRangeJoin(LogicalOperator &op, PhysicalOperatorType t
167
167
  // TODO: use stats to improve the choice?
168
168
  // TODO: Prefer fixed length types?
169
169
  if (conditions.size() > 1) {
170
- auto conditions_p = std::move(conditions);
171
- conditions.resize(conditions_p.size());
170
+ vector<JoinCondition> conditions_p(conditions.size());
171
+ std::swap(conditions_p, conditions);
172
172
  idx_t range_position = 0;
173
173
  idx_t other_position = conditions_p.size();
174
174
  for (idx_t i = 0; i < conditions_p.size(); ++i) {
@@ -13,7 +13,7 @@ CSVFileHandle::CSVFileHandle(FileSystem &fs, Allocator &allocator, unique_ptr<Fi
13
13
 
14
14
  unique_ptr<FileHandle> CSVFileHandle::OpenFileHandle(FileSystem &fs, Allocator &allocator, const string &path,
15
15
  FileCompressionType compression) {
16
- auto file_handle = fs.OpenFile(path.c_str(), FileFlags::FILE_FLAGS_READ, FileLockType::NO_LOCK, compression);
16
+ auto file_handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_READ, FileLockType::NO_LOCK, compression);
17
17
  if (file_handle->CanSeek()) {
18
18
  file_handle->Reset();
19
19
  }
@@ -41,7 +41,7 @@ PhysicalInsert::PhysicalInsert(vector<LogicalType> types_p, TableCatalogEntry &t
41
41
  return;
42
42
  }
43
43
 
44
- D_ASSERT(set_expressions.size() == set_columns.size());
44
+ D_ASSERT(this->set_expressions.size() == this->set_columns.size());
45
45
 
46
46
  // One or more columns are referenced from the existing table,
47
47
  // we use the 'insert_types' to figure out which types these columns have