duckdb 0.8.2-dev1182.0 → 0.8.2-dev1435.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/binding.gyp +7 -7
  2. package/package.json +1 -1
  3. package/src/duckdb/extension/parquet/parquet_extension.cpp +1 -1
  4. package/src/duckdb/extension/parquet/parquet_reader.cpp +1 -1
  5. package/src/duckdb/src/common/arrow/arrow_appender.cpp +0 -1
  6. package/src/duckdb/src/common/exception.cpp +2 -2
  7. package/src/duckdb/src/common/file_system.cpp +4 -0
  8. package/src/duckdb/src/common/hive_partitioning.cpp +10 -6
  9. package/src/duckdb/src/common/local_file_system.cpp +1 -1
  10. package/src/duckdb/src/common/multi_file_reader.cpp +3 -2
  11. package/src/duckdb/src/common/operator/cast_operators.cpp +35 -1
  12. package/src/duckdb/src/common/re2_regex.cpp +1 -1
  13. package/src/duckdb/src/common/types/bit.cpp +51 -0
  14. package/src/duckdb/src/common/types/uuid.cpp +2 -2
  15. package/src/duckdb/src/common/virtual_file_system.cpp +138 -1
  16. package/src/duckdb/src/core_functions/aggregate/holistic/reservoir_quantile.cpp +2 -0
  17. package/src/duckdb/src/core_functions/function_list.cpp +2 -2
  18. package/src/duckdb/src/core_functions/scalar/date/date_part.cpp +2 -2
  19. package/src/duckdb/src/core_functions/scalar/date/epoch.cpp +10 -7
  20. package/src/duckdb/src/execution/expression_executor.cpp +1 -1
  21. package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +3 -3
  22. package/src/duckdb/src/execution/operator/join/physical_asof_join.cpp +1 -1
  23. package/src/duckdb/src/execution/operator/join/physical_range_join.cpp +2 -2
  24. package/src/duckdb/src/execution/operator/persistent/csv_file_handle.cpp +1 -1
  25. package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +1 -1
  26. package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp +7 -2
  27. package/src/duckdb/src/execution/physical_plan/plan_get.cpp +2 -2
  28. package/src/duckdb/src/execution/window_segment_tree.cpp +1 -1
  29. package/src/duckdb/src/function/cast/bit_cast.cpp +34 -2
  30. package/src/duckdb/src/function/cast/blob_cast.cpp +3 -0
  31. package/src/duckdb/src/function/cast/numeric_casts.cpp +2 -0
  32. package/src/duckdb/src/function/function.cpp +1 -1
  33. package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
  34. package/src/duckdb/src/include/duckdb/common/extra_operator_info.hpp +27 -0
  35. package/src/duckdb/src/include/duckdb/common/file_system.hpp +2 -0
  36. package/src/duckdb/src/include/duckdb/common/hive_partitioning.hpp +1 -1
  37. package/src/duckdb/src/include/duckdb/common/operator/cast_operators.hpp +43 -3
  38. package/src/duckdb/src/include/duckdb/common/operator/numeric_cast.hpp +10 -0
  39. package/src/duckdb/src/include/duckdb/common/types/bit.hpp +81 -0
  40. package/src/duckdb/src/include/duckdb/common/virtual_file_system.hpp +38 -97
  41. package/src/duckdb/src/include/duckdb/core_functions/scalar/date_functions.hpp +11 -11
  42. package/src/duckdb/src/include/duckdb/execution/operator/persistent/base_csv_reader.hpp +1 -1
  43. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp +5 -1
  44. package/src/duckdb/src/include/duckdb/main/connection.hpp +1 -2
  45. package/src/duckdb/src/include/duckdb/main/relation/cross_product_relation.hpp +4 -1
  46. package/src/duckdb/src/include/duckdb/main/relation/join_relation.hpp +5 -2
  47. package/src/duckdb/src/include/duckdb/main/relation.hpp +4 -2
  48. package/src/duckdb/src/include/duckdb/main/settings.hpp +9 -0
  49. package/src/duckdb/src/include/duckdb/optimizer/join_order/cardinality_estimator.hpp +1 -1
  50. package/src/duckdb/src/include/duckdb/planner/constraints/bound_unique_constraint.hpp +3 -3
  51. package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp +4 -0
  52. package/src/duckdb/src/main/capi/duckdb_value-c.cpp +1 -1
  53. package/src/duckdb/src/main/config.cpp +1 -0
  54. package/src/duckdb/src/main/database.cpp +1 -1
  55. package/src/duckdb/src/main/extension/extension_install.cpp +6 -0
  56. package/src/duckdb/src/main/extension/extension_load.cpp +10 -1
  57. package/src/duckdb/src/main/relation/cross_product_relation.cpp +4 -3
  58. package/src/duckdb/src/main/relation/join_relation.cpp +5 -5
  59. package/src/duckdb/src/main/relation.cpp +6 -5
  60. package/src/duckdb/src/main/settings/settings.cpp +24 -0
  61. package/src/duckdb/src/parser/parser.cpp +8 -2
  62. package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +5 -4
  63. package/src/duckdb/src/planner/expression/bound_aggregate_expression.cpp +1 -1
  64. package/src/duckdb/src/planner/operator/logical_get.cpp +9 -4
  65. package/src/duckdb/src/storage/compression/rle.cpp +0 -1
  66. package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +6 -6
  67. package/src/statement.cpp +2 -1
@@ -16,17 +16,18 @@ PhysicalTableScan::PhysicalTableScan(vector<LogicalType> types, TableFunction fu
16
16
  : PhysicalOperator(PhysicalOperatorType::TABLE_SCAN, std::move(types), estimated_cardinality),
17
17
  function(std::move(function_p)), bind_data(std::move(bind_data_p)), column_ids(std::move(column_ids_p)),
18
18
  names(std::move(names_p)), table_filters(std::move(table_filters_p)) {
19
+ extra_info.file_filters = "";
19
20
  }
20
21
 
21
22
  PhysicalTableScan::PhysicalTableScan(vector<LogicalType> types, TableFunction function_p,
22
23
  unique_ptr<FunctionData> bind_data_p, vector<LogicalType> returned_types_p,
23
24
  vector<column_t> column_ids_p, vector<idx_t> projection_ids_p,
24
25
  vector<string> names_p, unique_ptr<TableFilterSet> table_filters_p,
25
- idx_t estimated_cardinality)
26
+ idx_t estimated_cardinality, ExtraOperatorInfo extra_info)
26
27
  : PhysicalOperator(PhysicalOperatorType::TABLE_SCAN, std::move(types), estimated_cardinality),
27
28
  function(std::move(function_p)), bind_data(std::move(bind_data_p)), returned_types(std::move(returned_types_p)),
28
29
  column_ids(std::move(column_ids_p)), projection_ids(std::move(projection_ids_p)), names(std::move(names_p)),
29
- table_filters(std::move(table_filters_p)) {
30
+ table_filters(std::move(table_filters_p)), extra_info(extra_info) {
30
31
  }
31
32
 
32
33
  class TableScanGlobalSourceState : public GlobalSourceState {
@@ -149,6 +150,10 @@ string PhysicalTableScan::ParamsToString() const {
149
150
  }
150
151
  }
151
152
  }
153
+ if (!extra_info.file_filters.empty()) {
154
+ result += "\n[INFOSEPARATOR]\n";
155
+ result += "File Filters: " + extra_info.file_filters;
156
+ }
152
157
  result += "\n[INFOSEPARATOR]\n";
153
158
  result += StringUtil::Format("EC: %llu", estimated_props->GetCardinality<idx_t>());
154
159
  return result;
@@ -55,7 +55,7 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalGet &op) {
55
55
  // function does not support projection pushdown
56
56
  auto node = make_uniq<PhysicalTableScan>(op.returned_types, op.function, std::move(op.bind_data),
57
57
  op.returned_types, op.column_ids, vector<column_t>(), op.names,
58
- std::move(table_filters), op.estimated_cardinality);
58
+ std::move(table_filters), op.estimated_cardinality, op.extra_info);
59
59
  // first check if an additional projection is necessary
60
60
  if (op.column_ids.size() == op.returned_types.size()) {
61
61
  bool projection_necessary = false;
@@ -93,7 +93,7 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalGet &op) {
93
93
  } else {
94
94
  return make_uniq<PhysicalTableScan>(op.types, op.function, std::move(op.bind_data), op.returned_types,
95
95
  op.column_ids, op.projection_ids, op.names, std::move(table_filters),
96
- op.estimated_cardinality);
96
+ op.estimated_cardinality, op.extra_info);
97
97
  }
98
98
  }
99
99
 
@@ -14,7 +14,7 @@ namespace duckdb {
14
14
  WindowAggregateState::WindowAggregateState(AggregateObject aggr, const LogicalType &result_type_p,
15
15
  idx_t partition_count_p)
16
16
  : aggr(std::move(aggr)), result_type(result_type_p), partition_count(partition_count_p),
17
- state_size(aggr.function.state_size()), state(state_size),
17
+ state_size(this->aggr.function.state_size()), state(state_size),
18
18
  statef(Value::POINTER(CastPointerToValue(state.data()))), filter_pos(0),
19
19
  allocator(Allocator::DefaultAllocator()) {
20
20
  statef.SetVectorType(VectorType::FLAT_VECTOR); // Prevent conversion of results to constants
@@ -1,3 +1,6 @@
1
+ #include "duckdb/common/hugeint.hpp"
2
+ #include "duckdb/common/operator/cast_operators.hpp"
3
+ #include "duckdb/common/types.hpp"
1
4
  #include "duckdb/function/cast/default_casts.hpp"
2
5
  #include "duckdb/function/cast/vector_cast_helpers.hpp"
3
6
 
@@ -6,9 +9,38 @@ namespace duckdb {
6
9
  BoundCastInfo DefaultCasts::BitCastSwitch(BindCastInput &input, const LogicalType &source, const LogicalType &target) {
7
10
  // now switch on the result type
8
11
  switch (target.id()) {
12
+ // Numerics
13
+ case LogicalTypeId::BOOLEAN:
14
+ return BoundCastInfo(&VectorCastHelpers::TryCastLoop<string_t, bool, CastFromBitToNumeric>);
15
+ case LogicalTypeId::TINYINT:
16
+ return BoundCastInfo(&VectorCastHelpers::TryCastLoop<string_t, int8_t, CastFromBitToNumeric>);
17
+ case LogicalTypeId::SMALLINT:
18
+ return BoundCastInfo(&VectorCastHelpers::TryCastLoop<string_t, int16_t, CastFromBitToNumeric>);
19
+ case LogicalTypeId::INTEGER:
20
+ return BoundCastInfo(&VectorCastHelpers::TryCastLoop<string_t, int32_t, CastFromBitToNumeric>);
21
+ case LogicalTypeId::BIGINT:
22
+ return BoundCastInfo(&VectorCastHelpers::TryCastLoop<string_t, int64_t, CastFromBitToNumeric>);
23
+ case LogicalTypeId::UTINYINT:
24
+ return BoundCastInfo(&VectorCastHelpers::TryCastLoop<string_t, uint8_t, CastFromBitToNumeric>);
25
+ case LogicalTypeId::USMALLINT:
26
+ return BoundCastInfo(&VectorCastHelpers::TryCastLoop<string_t, uint16_t, CastFromBitToNumeric>);
27
+ case LogicalTypeId::UINTEGER:
28
+ return BoundCastInfo(&VectorCastHelpers::TryCastLoop<string_t, uint32_t, CastFromBitToNumeric>);
29
+ case LogicalTypeId::UBIGINT:
30
+ return BoundCastInfo(&VectorCastHelpers::TryCastLoop<string_t, uint64_t, CastFromBitToNumeric>);
31
+ case LogicalTypeId::HUGEINT:
32
+ return BoundCastInfo(&VectorCastHelpers::TryCastLoop<string_t, hugeint_t, CastFromBitToNumeric>);
33
+ case LogicalTypeId::FLOAT:
34
+ return BoundCastInfo(&VectorCastHelpers::TryCastLoop<string_t, float, CastFromBitToNumeric>);
35
+ case LogicalTypeId::DOUBLE:
36
+ return BoundCastInfo(&VectorCastHelpers::TryCastLoop<string_t, double, CastFromBitToNumeric>);
37
+
38
+ case LogicalTypeId::BLOB:
39
+ return BoundCastInfo(&VectorCastHelpers::StringCast<string_t, CastFromBitToBlob>);
40
+
9
41
  case LogicalTypeId::VARCHAR:
10
- // bit to varchar
11
- return BoundCastInfo(&VectorCastHelpers::StringCast<string_t, duckdb::CastFromBit>);
42
+ return BoundCastInfo(&VectorCastHelpers::StringCast<string_t, CastFromBitToString>);
43
+
12
44
  default:
13
45
  return DefaultCasts::TryVectorNullCast;
14
46
  }
@@ -11,6 +11,9 @@ BoundCastInfo DefaultCasts::BlobCastSwitch(BindCastInput &input, const LogicalTy
11
11
  return BoundCastInfo(&VectorCastHelpers::StringCast<string_t, duckdb::CastFromBlob>);
12
12
  case LogicalTypeId::AGGREGATE_STATE:
13
13
  return DefaultCasts::ReinterpretCast;
14
+ case LogicalTypeId::BIT:
15
+ return BoundCastInfo(&VectorCastHelpers::StringCast<string_t, duckdb::CastFromBlobToBit>);
16
+
14
17
  default:
15
18
  return DefaultCasts::TryVectorNullCast;
16
19
  }
@@ -37,6 +37,8 @@ static BoundCastInfo InternalNumericCastSwitch(const LogicalType &source, const
37
37
  return BoundCastInfo(&VectorCastHelpers::ToDecimalCast<SRC>);
38
38
  case LogicalTypeId::VARCHAR:
39
39
  return BoundCastInfo(&VectorCastHelpers::StringCast<SRC, duckdb::StringCast>);
40
+ case LogicalTypeId::BIT:
41
+ return BoundCastInfo(&VectorCastHelpers::StringCast<SRC, duckdb::NumericTryCastToBit>);
40
42
  default:
41
43
  return DefaultCasts::TryVectorNullCast;
42
44
  }
@@ -113,7 +113,7 @@ void BuiltinFunctions::Initialize() {
113
113
  hash_t BaseScalarFunction::Hash() const {
114
114
  hash_t hash = return_type.Hash();
115
115
  for (auto &arg : arguments) {
116
- duckdb::CombineHash(hash, arg.Hash());
116
+ hash = duckdb::CombineHash(hash, arg.Hash());
117
117
  }
118
118
  return hash;
119
119
  }
@@ -1,8 +1,8 @@
1
1
  #ifndef DUCKDB_VERSION
2
- #define DUCKDB_VERSION "0.8.2-dev1182"
2
+ #define DUCKDB_VERSION "0.8.2-dev1435"
3
3
  #endif
4
4
  #ifndef DUCKDB_SOURCE_ID
5
- #define DUCKDB_SOURCE_ID "c4b64ad14e"
5
+ #define DUCKDB_SOURCE_ID "50d861cee9"
6
6
  #endif
7
7
  #include "duckdb/function/table/system_functions.hpp"
8
8
  #include "duckdb/main/database.hpp"
@@ -0,0 +1,27 @@
1
+ //===----------------------------------------------------------------------===//
2
+ // DuckDB
3
+ //
4
+ // duckdb/common/extra_operator_info.hpp
5
+ //
6
+ //
7
+ //===----------------------------------------------------------------------===//
8
+
9
+ #pragma once
10
+
11
+ #include <algorithm>
12
+ #include <cstdint>
13
+ #include <cstring>
14
+ #include "duckdb/common/operator/comparison_operators.hpp"
15
+
16
+ namespace duckdb {
17
+
18
+ class ExtraOperatorInfo {
19
+ public:
20
+ ExtraOperatorInfo() : file_filters("") {
21
+ }
22
+ ExtraOperatorInfo(ExtraOperatorInfo &extra_info) : file_filters(extra_info.file_filters) {
23
+ }
24
+ string file_filters;
25
+ };
26
+
27
+ } // namespace duckdb
@@ -246,6 +246,8 @@ public:
246
246
 
247
247
  //! Whether or not a file is remote or local, based only on file path
248
248
  DUCKDB_API static bool IsRemoteFile(const string &path);
249
+
250
+ DUCKDB_API virtual void SetDisabledFileSystems(const vector<string> &names);
249
251
  };
250
252
 
251
253
  } // namespace duckdb
@@ -31,7 +31,7 @@ public:
31
31
  //! evaluate to true.
32
32
  DUCKDB_API static void ApplyFiltersToFileList(ClientContext &context, vector<string> &files,
33
33
  vector<unique_ptr<Expression>> &filters,
34
- unordered_map<string, column_t> &column_map, idx_t table_index,
34
+ unordered_map<string, column_t> &column_map, LogicalGet &get,
35
35
  bool hive_enabled, bool filename_enabled);
36
36
 
37
37
  //! Returns the compiled regex pattern to match hive partitions
@@ -8,13 +8,18 @@
8
8
 
9
9
  #pragma once
10
10
 
11
+ #include "duckdb/common/assert.hpp"
11
12
  #include "duckdb/common/constants.hpp"
13
+ #include "duckdb/common/hugeint.hpp"
12
14
  #include "duckdb/common/limits.hpp"
13
15
  #include "duckdb/common/exception.hpp"
16
+ #include "duckdb/common/typedefs.hpp"
14
17
  #include "duckdb/common/types/string_type.hpp"
15
18
  #include "duckdb/common/types.hpp"
16
19
  #include "duckdb/common/operator/convert_to_string.hpp"
17
20
  #include "duckdb/common/types/null_value.hpp"
21
+ #include "duckdb/common/types/bit.hpp"
22
+ #include "duckdb/common/types/vector.hpp"
18
23
 
19
24
  namespace duckdb {
20
25
  struct ValidityMask;
@@ -652,6 +657,15 @@ struct CastFromBlob {
652
657
  template <>
653
658
  duckdb::string_t CastFromBlob::Operation(duckdb::string_t input, Vector &vector);
654
659
 
660
+ struct CastFromBlobToBit {
661
+ template <class SRC>
662
+ static inline string_t Operation(SRC input, Vector &result) {
663
+ throw NotImplementedException("Cast from blob could not be performed!");
664
+ }
665
+ };
666
+ template <>
667
+ string_t CastFromBlobToBit::Operation(string_t input, Vector &result);
668
+
655
669
  struct TryCastToBlob {
656
670
  template <class SRC, class DST>
657
671
  static inline bool Operation(SRC input, DST &result, Vector &result_vector, string *error_message,
@@ -659,7 +673,6 @@ struct TryCastToBlob {
659
673
  throw InternalException("Unsupported type for try cast to blob");
660
674
  }
661
675
  };
662
-
663
676
  template <>
664
677
  bool TryCastToBlob::Operation(string_t input, string_t &result, Vector &result_vector, string *error_message,
665
678
  bool strict);
@@ -667,14 +680,41 @@ bool TryCastToBlob::Operation(string_t input, string_t &result, Vector &result_v
667
680
  //===--------------------------------------------------------------------===//
668
681
  // Bits
669
682
  //===--------------------------------------------------------------------===//
670
- struct CastFromBit {
683
+ struct CastFromBitToString {
671
684
  template <class SRC>
672
685
  static inline string_t Operation(SRC input, Vector &result) {
673
686
  throw duckdb::NotImplementedException("Cast from bit could not be performed!");
674
687
  }
675
688
  };
676
689
  template <>
677
- duckdb::string_t CastFromBit::Operation(duckdb::string_t input, Vector &vector);
690
+ duckdb::string_t CastFromBitToString::Operation(duckdb::string_t input, Vector &vector);
691
+
692
+ struct CastFromBitToNumeric {
693
+ template <class SRC = string_t, class DST>
694
+ static inline bool Operation(SRC input, DST &result, bool strict = false) {
695
+ D_ASSERT(input.GetSize() > 1);
696
+
697
+ // TODO: Allow conversion if the significant bytes of the bitstring can be cast to the target type
698
+ // Currently only allows bitstring -> numeric if the full bitstring fits inside the numeric type
699
+ if (input.GetSize() - 1 > sizeof(DST)) {
700
+ throw ConversionException("Bitstring doesn't fit inside of %s", GetTypeId<DST>());
701
+ }
702
+ Bit::BitToNumeric(input, result);
703
+ return (true);
704
+ }
705
+ };
706
+ template <>
707
+ bool CastFromBitToNumeric::Operation(string_t input, bool &result, bool strict);
708
+ template <>
709
+ bool CastFromBitToNumeric::Operation(string_t input, hugeint_t &result, bool strict);
710
+
711
+ struct CastFromBitToBlob {
712
+ template <class SRC>
713
+ static inline string_t Operation(SRC input, Vector &result) {
714
+ D_ASSERT(input.GetSize() > 1);
715
+ return StringVector::AddStringOrBlob(result, Bit::BitToBlob(input));
716
+ }
717
+ };
678
718
 
679
719
  struct TryCastToBit {
680
720
  template <class SRC, class DST>
@@ -9,8 +9,11 @@
9
9
  #pragma once
10
10
 
11
11
  #include "duckdb/common/operator/cast_operators.hpp"
12
+ #include "duckdb/common/types/bit.hpp"
12
13
  #include "duckdb/common/types/hugeint.hpp"
14
+ #include "duckdb/common/types/string_type.hpp"
13
15
  #include "duckdb/common/types/value.hpp"
16
+ #include "duckdb/common/types/vector.hpp"
14
17
  #include <cmath>
15
18
 
16
19
  namespace duckdb {
@@ -442,6 +445,13 @@ bool TryCastWithOverflowCheck(hugeint_t value, double &result) {
442
445
  return Hugeint::TryCast(value, result);
443
446
  }
444
447
 
448
+ struct NumericTryCastToBit {
449
+ template <class SRC>
450
+ static inline string_t Operation(SRC input, Vector &result) {
451
+ return StringVector::AddStringOrBlob(result, Bit::NumericToBit(input));
452
+ }
453
+ };
454
+
445
455
  struct NumericTryCast {
446
456
  template <class SRC, class DST>
447
457
  static inline bool Operation(SRC input, DST &result, bool strict = false) {
@@ -8,8 +8,12 @@
8
8
 
9
9
  #pragma once
10
10
 
11
+ #include "duckdb/common/assert.hpp"
11
12
  #include "duckdb/common/common.hpp"
13
+ #include "duckdb/common/hugeint.hpp"
14
+ #include "duckdb/common/limits.hpp"
12
15
  #include "duckdb/common/types.hpp"
16
+ #include "duckdb/common/types/string_type.hpp"
13
17
 
14
18
  namespace duckdb {
15
19
 
@@ -37,7 +41,33 @@ public:
37
41
  //! Convert a string to a bit. This function should ONLY be called after calling GetBitSize, since it does NOT
38
42
  //! perform data validation.
39
43
  DUCKDB_API static void ToBit(string_t str, string_t &output);
44
+
40
45
  DUCKDB_API static string ToBit(string_t str);
46
+
47
+ //! output needs to have enough space allocated before calling this function (blob size + 1)
48
+ DUCKDB_API static void BlobToBit(string_t blob, string_t &output);
49
+
50
+ DUCKDB_API static string BlobToBit(string_t blob);
51
+
52
+ //! output_str needs to have enough space allocated before calling this function (sizeof(T) + 1)
53
+ template <class T>
54
+ static void NumericToBit(T numeric, string_t &output_str);
55
+
56
+ template <class T>
57
+ static string NumericToBit(T numeric);
58
+
59
+ //! bit is expected to fit inside of output num (bit size <= sizeof(T) + 1)
60
+ template <class T>
61
+ static void BitToNumeric(string_t bit, T &output_num);
62
+
63
+ template <class T>
64
+ static T BitToNumeric(string_t bit);
65
+
66
+ //! bit is expected to fit inside of output_blob (bit size = output_blob + 1)
67
+ static void BitToBlob(string_t bit, string_t &output_blob);
68
+
69
+ static string BitToBlob(string_t bit);
70
+
41
71
  //! Creates a new bitstring of determined length
42
72
  DUCKDB_API static void BitString(const string_t &input, const idx_t &len, string_t &result);
43
73
  DUCKDB_API static void SetEmptyBitString(string_t &target, string_t &input);
@@ -58,5 +88,56 @@ private:
58
88
  static idx_t GetBitInternal(string_t bit_string, idx_t n);
59
89
  static void SetBitInternal(string_t &bit_string, idx_t n, idx_t new_value);
60
90
  static idx_t GetBitIndex(idx_t n);
91
+ static uint8_t GetFirstByte(const string_t &str);
61
92
  };
93
+
94
+ //===--------------------------------------------------------------------===//
95
+ // Bit Template definitions
96
+ //===--------------------------------------------------------------------===//
97
+ template <class T>
98
+ void Bit::NumericToBit(T numeric, string_t &output_str) {
99
+ D_ASSERT(output_str.GetSize() >= sizeof(T) + 1);
100
+
101
+ auto output = output_str.GetDataWriteable();
102
+ auto data = const_data_ptr_cast(&numeric);
103
+
104
+ *output = 0; // set padding to 0
105
+ ++output;
106
+ for (idx_t idx = 0; idx < sizeof(T); ++idx) {
107
+ output[idx] = data[sizeof(T) - idx - 1];
108
+ }
109
+ Bit::Finalize(output_str);
110
+ }
111
+
112
+ template <class T>
113
+ string Bit::NumericToBit(T numeric) {
114
+ auto bit_len = sizeof(T) + 1;
115
+ auto buffer = make_unsafe_uniq_array<char>(bit_len);
116
+ string_t output_str(buffer.get(), bit_len);
117
+ Bit::NumericToBit(numeric, output_str);
118
+ return output_str.GetString();
119
+ }
120
+
121
+ template <class T>
122
+ T Bit::BitToNumeric(string_t bit) {
123
+ T output;
124
+ Bit::BitToNumeric(bit, output);
125
+ return (output);
126
+ }
127
+
128
+ template <class T>
129
+ void Bit::BitToNumeric(string_t bit, T &output_num) {
130
+ D_ASSERT(bit.GetSize() <= sizeof(T) + 1);
131
+
132
+ output_num = 0;
133
+ auto data = const_data_ptr_cast(bit.GetData());
134
+ auto output = data_ptr_cast(&output_num);
135
+
136
+ idx_t padded_byte_idx = sizeof(T) - bit.GetSize() + 1;
137
+ output[sizeof(T) - 1 - padded_byte_idx] = GetFirstByte(bit);
138
+ for (idx_t idx = padded_byte_idx + 1; idx < sizeof(T); ++idx) {
139
+ output[sizeof(T) - 1 - idx] = data[1 + idx - padded_byte_idx];
140
+ }
141
+ }
142
+
62
143
  } // namespace duckdb
@@ -10,6 +10,7 @@
10
10
 
11
11
  #include "duckdb/common/file_system.hpp"
12
12
  #include "duckdb/common/map.hpp"
13
+ #include "duckdb/common/unordered_set.hpp"
13
14
 
14
15
  namespace duckdb {
15
16
 
@@ -22,120 +23,60 @@ public:
22
23
  FileCompressionType compression = FileCompressionType::UNCOMPRESSED,
23
24
  FileOpener *opener = nullptr) override;
24
25
 
25
- void Read(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) override {
26
- handle.file_system.Read(handle, buffer, nr_bytes, location);
27
- };
26
+ void Read(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) override;
27
+ void Write(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) override;
28
28
 
29
- void Write(FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) override {
30
- handle.file_system.Write(handle, buffer, nr_bytes, location);
31
- }
29
+ int64_t Read(FileHandle &handle, void *buffer, int64_t nr_bytes) override;
32
30
 
33
- int64_t Read(FileHandle &handle, void *buffer, int64_t nr_bytes) override {
34
- return handle.file_system.Read(handle, buffer, nr_bytes);
35
- }
31
+ int64_t Write(FileHandle &handle, void *buffer, int64_t nr_bytes) override;
36
32
 
37
- int64_t Write(FileHandle &handle, void *buffer, int64_t nr_bytes) override {
38
- return handle.file_system.Write(handle, buffer, nr_bytes);
39
- }
33
+ int64_t GetFileSize(FileHandle &handle) override;
34
+ time_t GetLastModifiedTime(FileHandle &handle) override;
35
+ FileType GetFileType(FileHandle &handle) override;
40
36
 
41
- int64_t GetFileSize(FileHandle &handle) override {
42
- return handle.file_system.GetFileSize(handle);
43
- }
44
- time_t GetLastModifiedTime(FileHandle &handle) override {
45
- return handle.file_system.GetLastModifiedTime(handle);
46
- }
47
- FileType GetFileType(FileHandle &handle) override {
48
- return handle.file_system.GetFileType(handle);
49
- }
37
+ void Truncate(FileHandle &handle, int64_t new_size) override;
50
38
 
51
- void Truncate(FileHandle &handle, int64_t new_size) override {
52
- handle.file_system.Truncate(handle, new_size);
53
- }
54
-
55
- void FileSync(FileHandle &handle) override {
56
- handle.file_system.FileSync(handle);
57
- }
39
+ void FileSync(FileHandle &handle) override;
58
40
 
59
41
  // need to look up correct fs for this
60
- bool DirectoryExists(const string &directory) override {
61
- return FindFileSystem(directory)->DirectoryExists(directory);
62
- }
63
- void CreateDirectory(const string &directory) override {
64
- FindFileSystem(directory)->CreateDirectory(directory);
65
- }
42
+ bool DirectoryExists(const string &directory) override;
43
+ void CreateDirectory(const string &directory) override;
66
44
 
67
- void RemoveDirectory(const string &directory) override {
68
- FindFileSystem(directory)->RemoveDirectory(directory);
69
- }
45
+ void RemoveDirectory(const string &directory) override;
70
46
 
71
47
  bool ListFiles(const string &directory, const std::function<void(const string &, bool)> &callback,
72
- FileOpener *opener = nullptr) override {
73
- return FindFileSystem(directory)->ListFiles(directory, callback, opener);
74
- }
75
-
76
- void MoveFile(const string &source, const string &target) override {
77
- FindFileSystem(source)->MoveFile(source, target);
78
- }
79
-
80
- bool FileExists(const string &filename) override {
81
- return FindFileSystem(filename)->FileExists(filename);
82
- }
83
-
84
- bool IsPipe(const string &filename) override {
85
- return FindFileSystem(filename)->IsPipe(filename);
86
- }
87
- virtual void RemoveFile(const string &filename) override {
88
- FindFileSystem(filename)->RemoveFile(filename);
89
- }
90
-
91
- virtual vector<string> Glob(const string &path, FileOpener *opener = nullptr) override {
92
- return FindFileSystem(path)->Glob(path, opener);
93
- }
94
-
95
- void RegisterSubSystem(unique_ptr<FileSystem> fs) override {
96
- sub_systems.push_back(std::move(fs));
97
- }
98
-
99
- void UnregisterSubSystem(const string &name) override {
100
- for (auto sub_system = sub_systems.begin(); sub_system != sub_systems.end(); sub_system++) {
101
- if (sub_system->get()->GetName() == name) {
102
- sub_systems.erase(sub_system);
103
- return;
104
- }
105
- }
106
- throw InvalidInputException("Could not find filesystem with name %s", name);
107
- }
108
-
109
- void RegisterSubSystem(FileCompressionType compression_type, unique_ptr<FileSystem> fs) override {
110
- compressed_fs[compression_type] = std::move(fs);
111
- }
112
-
113
- vector<string> ListSubSystems() override {
114
- vector<string> names(sub_systems.size());
115
- for (idx_t i = 0; i < sub_systems.size(); i++) {
116
- names[i] = sub_systems[i]->GetName();
117
- }
118
- return names;
119
- }
120
-
121
- std::string GetName() const override {
122
- return "VirtualFileSystem";
123
- }
48
+ FileOpener *opener = nullptr) override;
49
+
50
+ void MoveFile(const string &source, const string &target) override;
51
+
52
+ bool FileExists(const string &filename) override;
53
+
54
+ bool IsPipe(const string &filename) override;
55
+ virtual void RemoveFile(const string &filename) override;
56
+
57
+ virtual vector<string> Glob(const string &path, FileOpener *opener = nullptr) override;
58
+
59
+ void RegisterSubSystem(unique_ptr<FileSystem> fs) override;
60
+
61
+ void UnregisterSubSystem(const string &name) override;
62
+
63
+ void RegisterSubSystem(FileCompressionType compression_type, unique_ptr<FileSystem> fs) override;
64
+
65
+ vector<string> ListSubSystems() override;
66
+
67
+ std::string GetName() const override;
68
+
69
+ void SetDisabledFileSystems(const vector<string> &names) override;
124
70
 
125
71
  private:
126
- FileSystem *FindFileSystem(const string &path) {
127
- for (auto &sub_system : sub_systems) {
128
- if (sub_system->CanHandleFile(path)) {
129
- return sub_system.get();
130
- }
131
- }
132
- return default_fs.get();
133
- }
72
+ FileSystem &FindFileSystem(const string &path);
73
+ FileSystem &FindFileSystemInternal(const string &path);
134
74
 
135
75
  private:
136
76
  vector<unique_ptr<FileSystem>> sub_systems;
137
77
  map<FileCompressionType, unique_ptr<FileSystem>> compressed_fs;
138
78
  const unique_ptr<FileSystem> default_fs;
79
+ unordered_set<string> disabled_file_systems;
139
80
  };
140
81
 
141
82
  } // namespace duckdb
@@ -162,8 +162,8 @@ struct DecadeFun {
162
162
 
163
163
  struct EpochFun {
164
164
  static constexpr const char *Name = "epoch";
165
- static constexpr const char *Parameters = "ts";
166
- static constexpr const char *Description = "Extract the epoch component from a date or timestamp";
165
+ static constexpr const char *Parameters = "temporal";
166
+ static constexpr const char *Description = "Extract the epoch component from a temporal type";
167
167
  static constexpr const char *Example = "epoch(timestamp '2021-08-03 11:59:44.123456')";
168
168
 
169
169
  static ScalarFunctionSet GetFunctions();
@@ -172,25 +172,25 @@ struct EpochFun {
172
172
  struct EpochMsFun {
173
173
  static constexpr const char *Name = "epoch_ms";
174
174
  static constexpr const char *Parameters = "temporal";
175
- static constexpr const char *Description = "Return the total number of milliseconds since the epoch";
175
+ static constexpr const char *Description = "Extract the epoch component in milliseconds from a temporal type";
176
176
  static constexpr const char *Example = "epoch_ms(timestamp '2021-08-03 11:59:44.123456')";
177
177
 
178
178
  static ScalarFunctionSet GetFunctions();
179
179
  };
180
180
 
181
- struct EpochMicrosecondsFun {
181
+ struct EpochUsFun {
182
182
  static constexpr const char *Name = "epoch_us";
183
183
  static constexpr const char *Parameters = "temporal";
184
- static constexpr const char *Description = "Return the total number of microseconds since the epoch";
184
+ static constexpr const char *Description = "Extract the epoch component in microseconds from a temporal type";
185
185
  static constexpr const char *Example = "epoch_us(timestamp '2021-08-03 11:59:44.123456')";
186
186
 
187
187
  static ScalarFunctionSet GetFunctions();
188
188
  };
189
189
 
190
- struct EpochNanosecondsFun {
190
+ struct EpochNsFun {
191
191
  static constexpr const char *Name = "epoch_ns";
192
192
  static constexpr const char *Parameters = "temporal";
193
- static constexpr const char *Description = "Return the total number of nanooseconds since the epoch";
193
+ static constexpr const char *Description = "Extract the epoch component in nanoseconds from a temporal type";
194
194
  static constexpr const char *Example = "epoch_ns(timestamp '2021-08-03 11:59:44.123456')";
195
195
 
196
196
  static ScalarFunctionSet GetFunctions();
@@ -291,7 +291,7 @@ struct MakeTimeFun {
291
291
 
292
292
  struct MakeTimestampFun {
293
293
  static constexpr const char *Name = "make_timestamp";
294
- static constexpr const char *Parameters = "year,month,day,hour,minute,seconds; or just microseconds since the epoch";
294
+ static constexpr const char *Parameters = "year,month,day,hour,minute,seconds";
295
295
  static constexpr const char *Description = "The timestamp for the given parts";
296
296
  static constexpr const char *Example = "make_timestamp(1992, 9, 20, 13, 34, 27.123456)";
297
297
 
@@ -490,8 +490,8 @@ struct ToSecondsFun {
490
490
  struct ToTimestampFun {
491
491
  static constexpr const char *Name = "to_timestamp";
492
492
  static constexpr const char *Parameters = "sec";
493
- static constexpr const char *Description = "Converts sec since epoch to a timestamp";
494
- static constexpr const char *Example = "to_timestamp(701222400)";
493
+ static constexpr const char *Description = "Converts secs since epoch to a timestamp with time zone";
494
+ static constexpr const char *Example = "to_timestamp(1284352323.5)";
495
495
 
496
496
  static ScalarFunction GetFunction();
497
497
  };
@@ -508,7 +508,7 @@ struct ToYearsFun {
508
508
  struct TryStrpTimeFun {
509
509
  static constexpr const char *Name = "try_strptime";
510
510
  static constexpr const char *Parameters = "text,format";
511
- static constexpr const char *Description = "Converts string to timestamp with time zone according to the format string if %Z is specified. Returns NULL on failure.";
511
+ static constexpr const char *Description = "Converts string to timestamp using the format string (timestamp with time zone if %Z is specified). Returns NULL on failure.";
512
512
  static constexpr const char *Example = "try_strptime('Wed, 1 January 1992 - 08:38:40 PM', '%a, %-d %B %Y - %I:%M:%S %p')";
513
513
 
514
514
  static ScalarFunctionSet GetFunctions();