duckdb 0.8.2-dev1859.0 → 0.8.2-dev1968.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/binding.gyp +9 -9
  2. package/package.json +1 -1
  3. package/src/duckdb/src/common/enum_util.cpp +28 -0
  4. package/src/duckdb/src/common/types/hugeint.cpp +40 -0
  5. package/src/duckdb/src/core_functions/function_list.cpp +1 -0
  6. package/src/duckdb/src/core_functions/scalar/string/to_base.cpp +66 -0
  7. package/src/duckdb/src/execution/operator/aggregate/physical_hash_aggregate.cpp +14 -11
  8. package/src/duckdb/src/execution/operator/aggregate/physical_perfecthash_aggregate.cpp +6 -4
  9. package/src/duckdb/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp +14 -12
  10. package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +6 -4
  11. package/src/duckdb/src/execution/operator/helper/physical_batch_collector.cpp +8 -6
  12. package/src/duckdb/src/execution/operator/helper/physical_explain_analyze.cpp +2 -2
  13. package/src/duckdb/src/execution/operator/helper/physical_limit.cpp +5 -3
  14. package/src/duckdb/src/execution/operator/helper/physical_materialized_collector.cpp +7 -5
  15. package/src/duckdb/src/execution/operator/helper/physical_vacuum.cpp +7 -5
  16. package/src/duckdb/src/execution/operator/join/physical_asof_join.cpp +5 -4
  17. package/src/duckdb/src/execution/operator/join/physical_blockwise_nl_join.cpp +2 -2
  18. package/src/duckdb/src/execution/operator/join/physical_delim_join.cpp +13 -6
  19. package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp +7 -5
  20. package/src/duckdb/src/execution/operator/join/physical_iejoin.cpp +7 -5
  21. package/src/duckdb/src/execution/operator/join/physical_nested_loop_join.cpp +7 -4
  22. package/src/duckdb/src/execution/operator/join/physical_piecewise_merge_join.cpp +8 -6
  23. package/src/duckdb/src/execution/operator/order/physical_order.cpp +7 -5
  24. package/src/duckdb/src/execution/operator/order/physical_top_n.cpp +7 -5
  25. package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp +8 -6
  26. package/src/duckdb/src/execution/operator/persistent/physical_batch_insert.cpp +8 -7
  27. package/src/duckdb/src/execution/operator/persistent/physical_copy_to_file.cpp +8 -6
  28. package/src/duckdb/src/execution/operator/persistent/physical_fixed_batch_copy.cpp +11 -9
  29. package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +10 -10
  30. package/src/duckdb/src/execution/operator/persistent/physical_update.cpp +4 -2
  31. package/src/duckdb/src/execution/operator/schema/physical_create_index.cpp +7 -6
  32. package/src/duckdb/src/execution/physical_operator.cpp +3 -2
  33. package/src/duckdb/src/function/table/read_csv.cpp +4 -3
  34. package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
  35. package/src/duckdb/src/include/duckdb/common/bitpacking.hpp +70 -55
  36. package/src/duckdb/src/include/duckdb/common/enum_util.hpp +8 -0
  37. package/src/duckdb/src/include/duckdb/common/enums/operator_result_type.hpp +5 -1
  38. package/src/duckdb/src/include/duckdb/common/hugeint.hpp +15 -0
  39. package/src/duckdb/src/include/duckdb/common/limits.hpp +52 -149
  40. package/src/duckdb/src/include/duckdb/common/numeric_utils.hpp +48 -0
  41. package/src/duckdb/src/include/duckdb/core_functions/aggregate/distributive_functions.hpp +1 -1
  42. package/src/duckdb/src/include/duckdb/core_functions/scalar/operators_functions.hpp +1 -1
  43. package/src/duckdb/src/include/duckdb/core_functions/scalar/string_functions.hpp +9 -0
  44. package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_hash_aggregate.hpp +3 -3
  45. package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_perfecthash_aggregate.hpp +1 -1
  46. package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_ungrouped_aggregate.hpp +3 -3
  47. package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_window.hpp +2 -2
  48. package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_batch_collector.hpp +2 -2
  49. package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_explain_analyze.hpp +1 -1
  50. package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_limit.hpp +1 -1
  51. package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_materialized_collector.hpp +1 -1
  52. package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_vacuum.hpp +2 -2
  53. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_asof_join.hpp +2 -2
  54. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_blockwise_nl_join.hpp +1 -1
  55. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_delim_join.hpp +2 -2
  56. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_hash_join.hpp +2 -2
  57. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_iejoin.hpp +2 -2
  58. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_nested_loop_join.hpp +2 -2
  59. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_piecewise_merge_join.hpp +2 -2
  60. package/src/duckdb/src/include/duckdb/execution/operator/order/physical_order.hpp +2 -2
  61. package/src/duckdb/src/include/duckdb/execution/operator/order/physical_top_n.hpp +2 -2
  62. package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_batch_copy_to_file.hpp +2 -2
  63. package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_batch_insert.hpp +2 -2
  64. package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_copy_to_file.hpp +2 -2
  65. package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_fixed_batch_copy.hpp +2 -2
  66. package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_insert.hpp +2 -2
  67. package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_update.hpp +1 -1
  68. package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_create_index.hpp +2 -2
  69. package/src/duckdb/src/include/duckdb/execution/physical_operator.hpp +3 -3
  70. package/src/duckdb/src/include/duckdb/execution/physical_operator_states.hpp +11 -0
  71. package/src/duckdb/src/include/duckdb/main/extension_entries.hpp +3 -0
  72. package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp +0 -3
  73. package/src/duckdb/src/include/duckdb/parallel/pipeline_executor.hpp +3 -2
  74. package/src/duckdb/src/main/relation/join_relation.cpp +1 -1
  75. package/src/duckdb/src/parallel/pipeline.cpp +0 -17
  76. package/src/duckdb/src/parallel/pipeline_executor.cpp +26 -7
  77. package/src/duckdb/src/parallel/pipeline_finish_event.cpp +55 -1
  78. package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +13 -3
  79. package/src/duckdb/src/storage/compression/bitpacking.cpp +87 -63
  80. package/src/duckdb/src/storage/compression/bitpacking_hugeint.cpp +295 -0
  81. package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +6 -6
  82. package/src/duckdb/ub_src_core_functions_scalar_string.cpp +2 -0
  83. package/src/duckdb/ub_src_storage_compression.cpp +2 -0
@@ -44,9 +44,9 @@ public:
44
44
  public:
45
45
  // Sink interface
46
46
  SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
47
- void Combine(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate) const override;
47
+ SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
48
48
  SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
49
- GlobalSinkState &gstate) const override;
49
+ OperatorSinkFinalizeInput &input) const override;
50
50
 
51
51
  unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
52
52
  unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
@@ -68,7 +68,7 @@ private:
68
68
  SinkFinalizeType FinalizeDistinct(Pipeline &pipeline, Event &event, ClientContext &context,
69
69
  GlobalSinkState &gstate) const;
70
70
  //! Combine the distinct aggregates
71
- void CombineDistinct(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate) const;
71
+ void CombineDistinct(ExecutionContext &context, OperatorSinkCombineInput &input) const;
72
72
  //! Sink the distinct aggregates
73
73
  void SinkDistinct(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const;
74
74
  };
@@ -51,9 +51,9 @@ public:
51
51
  public:
52
52
  // Sink interface
53
53
  SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
54
- void Combine(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate) const override;
54
+ SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
55
55
  SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
56
- GlobalSinkState &gstate) const override;
56
+ OperatorSinkFinalizeInput &input) const override;
57
57
 
58
58
  unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
59
59
  unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
@@ -22,9 +22,9 @@ public:
22
22
  public:
23
23
  // Sink interface
24
24
  SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
25
- void Combine(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate) const override;
25
+ SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
26
26
  SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
27
- GlobalSinkState &gstate) const override;
27
+ OperatorSinkFinalizeInput &input) const override;
28
28
 
29
29
  unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
30
30
  unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
@@ -34,7 +34,7 @@ public:
34
34
  // Sink Interface
35
35
  SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
36
36
  SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
37
- GlobalSinkState &gstate) const override;
37
+ OperatorSinkFinalizeInput &input) const override;
38
38
 
39
39
  unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
40
40
 
@@ -44,7 +44,7 @@ public:
44
44
  public:
45
45
  // Sink Interface
46
46
  SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
47
- void Combine(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate) const override;
47
+ SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
48
48
  unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
49
49
  unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
50
50
 
@@ -24,7 +24,7 @@ public:
24
24
  public:
25
25
  // Sink interface
26
26
  SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
27
- void Combine(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate) const override;
27
+ SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
28
28
 
29
29
  unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
30
30
  unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
@@ -38,9 +38,9 @@ public:
38
38
  unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
39
39
 
40
40
  SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
41
- void Combine(ExecutionContext &context, GlobalSinkState &gstate_p, LocalSinkState &lstate_p) const override;
41
+ SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
42
42
  SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
43
- GlobalSinkState &gstate) const override;
43
+ OperatorSinkFinalizeInput &input) const override;
44
44
 
45
45
  bool IsSink() const override {
46
46
  return info->has_table;
@@ -69,9 +69,9 @@ public:
69
69
  unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
70
70
  unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
71
71
  SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
72
- void Combine(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate) const override;
72
+ SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
73
73
  SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
74
- GlobalSinkState &gstate) const override;
74
+ OperatorSinkFinalizeInput &input) const override;
75
75
 
76
76
  bool IsSink() const override {
77
77
  return true;
@@ -59,7 +59,7 @@ public:
59
59
  unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
60
60
  SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
61
61
  SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
62
- GlobalSinkState &gstate) const override;
62
+ OperatorSinkFinalizeInput &input) const override;
63
63
 
64
64
  bool IsSink() const override {
65
65
  return true;
@@ -35,9 +35,9 @@ public:
35
35
  unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
36
36
  unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
37
37
  SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
38
- void Combine(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate) const override;
38
+ SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
39
39
  SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
40
- GlobalSinkState &gstate) const override;
40
+ OperatorSinkFinalizeInput &input) const override;
41
41
 
42
42
  bool IsSink() const override {
43
43
  return true;
@@ -79,9 +79,9 @@ public:
79
79
 
80
80
  unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
81
81
  SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
82
- void Combine(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate) const override;
82
+ SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
83
83
  SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
84
- GlobalSinkState &gstate) const override;
84
+ OperatorSinkFinalizeInput &input) const override;
85
85
 
86
86
  bool IsSink() const override {
87
87
  return true;
@@ -51,9 +51,9 @@ public:
51
51
  unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
52
52
  unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
53
53
  SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
54
- void Combine(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate) const override;
54
+ SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
55
55
  SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
56
- GlobalSinkState &gstate) const override;
56
+ OperatorSinkFinalizeInput &input) const override;
57
57
 
58
58
  bool IsSink() const override {
59
59
  return true;
@@ -54,9 +54,9 @@ public:
54
54
  unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
55
55
  unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
56
56
  SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
57
- void Combine(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate) const override;
57
+ SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
58
58
  SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
59
- GlobalSinkState &gstate) const override;
59
+ OperatorSinkFinalizeInput &input) const override;
60
60
 
61
61
  bool IsSink() const override {
62
62
  return true;
@@ -60,9 +60,9 @@ public:
60
60
  unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
61
61
  unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
62
62
  SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
63
- void Combine(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate) const override;
63
+ SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
64
64
  SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
65
- GlobalSinkState &gstate) const override;
65
+ OperatorSinkFinalizeInput &input) const override;
66
66
 
67
67
  bool IsSink() const override {
68
68
  return true;
@@ -60,9 +60,9 @@ public:
60
60
  unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
61
61
  unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
62
62
  SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
63
- void Combine(ExecutionContext &context, GlobalSinkState &gstate_p, LocalSinkState &lstate_p) const override;
63
+ SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
64
64
  SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
65
- GlobalSinkState &gstate) const override;
65
+ OperatorSinkFinalizeInput &input) const override;
66
66
 
67
67
  bool IsSink() const override {
68
68
  return true;
@@ -42,9 +42,9 @@ public:
42
42
 
43
43
  public:
44
44
  SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
45
- void Combine(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate) const override;
45
+ SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
46
46
  SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
47
- GlobalSinkState &gstate) const override;
47
+ OperatorSinkFinalizeInput &input) const override;
48
48
  unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
49
49
  unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
50
50
 
@@ -41,9 +41,9 @@ public:
41
41
  public:
42
42
  // Sink interface
43
43
  SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
44
- void Combine(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate) const override;
44
+ SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
45
45
  SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
46
- GlobalSinkState &gstate) const override;
46
+ OperatorSinkFinalizeInput &input) const override;
47
47
  unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
48
48
  unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
49
49
  void NextBatch(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate_p) const override;
@@ -54,9 +54,9 @@ public:
54
54
  unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
55
55
  void NextBatch(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate_p) const override;
56
56
  SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
57
- void Combine(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate) const override;
57
+ SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
58
58
  SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
59
- GlobalSinkState &gstate) const override;
59
+ OperatorSinkFinalizeInput &input) const override;
60
60
 
61
61
  bool RequiresBatchIndex() const override {
62
62
  return true;
@@ -50,9 +50,9 @@ public:
50
50
  public:
51
51
  // Sink interface
52
52
  SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
53
- void Combine(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate) const override;
53
+ SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
54
54
  SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
55
- GlobalSinkState &gstate) const override;
55
+ OperatorSinkFinalizeInput &input) const override;
56
56
  unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
57
57
  unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
58
58
 
@@ -40,9 +40,9 @@ public:
40
40
  public:
41
41
  // Sink interface
42
42
  SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
43
- void Combine(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate) const override;
43
+ SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
44
44
  SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
45
- GlobalSinkState &gstate) const override;
45
+ OperatorSinkFinalizeInput &input) const override;
46
46
  unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
47
47
  unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
48
48
  void NextBatch(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate_p) const override;
@@ -88,9 +88,9 @@ public:
88
88
  unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
89
89
  unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
90
90
  SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
91
- void Combine(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate) const override;
91
+ SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
92
92
  SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
93
- GlobalSinkState &gstate) const override;
93
+ OperatorSinkFinalizeInput &input) const override;
94
94
 
95
95
  bool IsSink() const override {
96
96
  return true;
@@ -47,7 +47,7 @@ public:
47
47
  unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
48
48
  unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const override;
49
49
  SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
50
- void Combine(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate) const override;
50
+ SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
51
51
 
52
52
  bool IsSink() const override {
53
53
  return true;
@@ -53,9 +53,9 @@ public:
53
53
  unique_ptr<GlobalSinkState> GetGlobalSinkState(ClientContext &context) const override;
54
54
 
55
55
  SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override;
56
- void Combine(ExecutionContext &context, GlobalSinkState &gstate_p, LocalSinkState &lstate_p) const override;
56
+ SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override;
57
57
  SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
58
- GlobalSinkState &gstate) const override;
58
+ OperatorSinkFinalizeInput &input) const override;
59
59
 
60
60
  bool IsSink() const override {
61
61
  return true;
@@ -140,14 +140,14 @@ public:
140
140
  // The combine is called when a single thread has completed execution of its part of the pipeline, it is the final
141
141
  // time that a specific LocalSinkState is accessible. This method can be called in parallel while other Sink() or
142
142
  // Combine() calls are active on the same GlobalSinkState.
143
- virtual void Combine(ExecutionContext &context, GlobalSinkState &gstate, LocalSinkState &lstate) const;
143
+ virtual SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const;
144
144
  //! The finalize is called when ALL threads are finished execution. It is called only once per pipeline, and is
145
145
  //! entirely single threaded.
146
146
  //! If Finalize returns SinkResultType::FINISHED, the sink is marked as finished
147
147
  virtual SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context,
148
- GlobalSinkState &gstate) const;
148
+ OperatorSinkFinalizeInput &input) const;
149
149
  //! For sinks with RequiresBatchIndex set to true, when a new batch starts being processed this method is called
150
- //! This allows flushing of the current batch (e.g. to disk)
150
+ //! This allows flushing of the current batch (e.g. to disk) TODO: should this be able to block too?
151
151
  virtual void NextBatch(ExecutionContext &context, GlobalSinkState &state, LocalSinkState &lstate_p) const;
152
152
 
153
153
  virtual unique_ptr<LocalSinkState> GetLocalSinkState(ExecutionContext &context) const;
@@ -165,6 +165,17 @@ struct OperatorSourceInput {
165
165
  InterruptState &interrupt_state;
166
166
  };
167
167
 
168
+ struct OperatorSinkCombineInput {
169
+ GlobalSinkState &global_state;
170
+ LocalSinkState &local_state;
171
+ InterruptState &interrupt_state;
172
+ };
173
+
174
+ struct OperatorSinkFinalizeInput {
175
+ GlobalSinkState &global_state;
176
+ InterruptState &interrupt_state;
177
+ };
178
+
168
179
  // LCOV_EXCL_STOP
169
180
 
170
181
  } // namespace duckdb
@@ -32,6 +32,9 @@ static constexpr ExtensionEntry EXTENSION_FUNCTIONS[] = {{"->>", "json"},
32
32
  {"from_substrait_json", "substrait"},
33
33
  {"get_substrait", "substrait"},
34
34
  {"get_substrait_json", "substrait"},
35
+ {"iceberg_scan", "iceberg"},
36
+ {"iceberg_metadata", "iceberg"},
37
+ {"iceberg_snapshots", "iceberg"},
35
38
  {"icu_calendar_names", "icu"},
36
39
  {"icu_sort_key", "icu"},
37
40
  {"json", "json"},
@@ -71,9 +71,6 @@ public:
71
71
  void ClearSource();
72
72
  void Schedule(shared_ptr<Event> &event);
73
73
 
74
- //! Finalize this pipeline
75
- void Finalize(Event &event);
76
-
77
74
  string ToString() const;
78
75
  void Print() const;
79
76
  void PrintDependencies() const;
@@ -48,8 +48,8 @@ public:
48
48
  //! If OperatorResultType::FINISHED is returned, more input will not change the result anymore
49
49
  OperatorResultType ExecutePush(DataChunk &input);
50
50
  //! Called after depleting the source: finalizes the execution of this pipeline executor
51
- //! This should only be called once per PipelineExecutor
52
- void PushFinalize();
51
+ //! This should only be called once per PipelineExecutor.
52
+ PipelineExecuteResult PushFinalize();
53
53
 
54
54
  //! Initializes a chunk with the types that will flow out of ExecutePull
55
55
  void InitializeChunk(DataChunk &chunk);
@@ -142,6 +142,7 @@ private:
142
142
  //! Debugging state: number of times blocked
143
143
  int debug_blocked_sink_count = 0;
144
144
  int debug_blocked_source_count = 0;
145
+ int debug_blocked_combine_count = 0;
145
146
  //! Number of times the Sink/Source will block before actually returning data
146
147
  int debug_blocked_target_count = 1;
147
148
  #endif
@@ -52,7 +52,7 @@ const vector<ColumnDefinition> &JoinRelation::Columns() {
52
52
 
53
53
  string JoinRelation::ToString(idx_t depth) {
54
54
  string str = RenderWhitespace(depth);
55
- str += "Join " + EnumUtil::ToString(join_type);
55
+ str += "Join " + EnumUtil::ToString(join_ref_type) + " " + EnumUtil::ToString(join_type);
56
56
  if (condition) {
57
57
  str += " " + condition->GetName();
58
58
  }
@@ -218,23 +218,6 @@ void Pipeline::Ready() {
218
218
  std::reverse(operators.begin(), operators.end());
219
219
  }
220
220
 
221
- void Pipeline::Finalize(Event &event) {
222
- if (executor.HasError()) {
223
- return;
224
- }
225
- D_ASSERT(ready);
226
- try {
227
- auto sink_state = sink->Finalize(*this, event, executor.context, *sink->sink_state);
228
- sink->sink_state->state = sink_state;
229
- } catch (Exception &ex) { // LCOV_EXCL_START
230
- executor.PushError(PreservedError(ex));
231
- } catch (std::exception &ex) {
232
- executor.PushError(PreservedError(ex));
233
- } catch (...) {
234
- executor.PushError(PreservedError("Unknown exception in Finalize!"));
235
- } // LCOV_EXCL_STOP
236
- }
237
-
238
221
  void Pipeline::AddDependency(shared_ptr<Pipeline> &pipeline) {
239
222
  D_ASSERT(pipeline);
240
223
  dependencies.push_back(weak_ptr<Pipeline>(pipeline));
@@ -171,9 +171,7 @@ PipelineExecuteResult PipelineExecutor::Execute(idx_t max_chunks) {
171
171
  return PipelineExecuteResult::NOT_FINISHED;
172
172
  }
173
173
 
174
- PushFinalize();
175
-
176
- return PipelineExecuteResult::FINISHED;
174
+ return PushFinalize();
177
175
  }
178
176
 
179
177
  PipelineExecuteResult PipelineExecutor::Execute() {
@@ -238,24 +236,45 @@ OperatorResultType PipelineExecutor::ExecutePushInternal(DataChunk &input, idx_t
238
236
  }
239
237
  }
240
238
 
241
- void PipelineExecutor::PushFinalize() {
239
+ PipelineExecuteResult PipelineExecutor::PushFinalize() {
242
240
  if (finalized) {
243
241
  throw InternalException("Calling PushFinalize on a pipeline that has been finalized already");
244
242
  }
245
243
 
246
244
  D_ASSERT(local_sink_state);
247
245
 
248
- finalized = true;
246
+ // Run the combine for the sink
247
+ OperatorSinkCombineInput combine_input {*pipeline.sink->sink_state, *local_sink_state, interrupt_state};
248
+
249
+ #ifdef DUCKDB_DEBUG_ASYNC_SINK_SOURCE
250
+ if (debug_blocked_combine_count < debug_blocked_target_count) {
251
+ debug_blocked_combine_count++;
252
+
253
+ auto &callback_state = combine_input.interrupt_state;
254
+ std::thread rewake_thread([callback_state] {
255
+ std::this_thread::sleep_for(std::chrono::milliseconds(1));
256
+ callback_state.Callback();
257
+ });
258
+ rewake_thread.detach();
249
259
 
250
- // run the combine for the sink
251
- pipeline.sink->Combine(context, *pipeline.sink->sink_state, *local_sink_state);
260
+ return PipelineExecuteResult::INTERRUPTED;
261
+ }
262
+ #endif
263
+ auto result = pipeline.sink->Combine(context, combine_input);
252
264
 
265
+ if (result == SinkCombineResultType::BLOCKED) {
266
+ return PipelineExecuteResult::INTERRUPTED;
267
+ }
268
+
269
+ finalized = true;
253
270
  // flush all query profiler info
254
271
  for (idx_t i = 0; i < intermediate_states.size(); i++) {
255
272
  intermediate_states[i]->Finalize(pipeline.operators[i].get(), context);
256
273
  }
257
274
  pipeline.executor.Flush(thread);
258
275
  local_sink_state.reset();
276
+
277
+ return PipelineExecuteResult::FINISHED;
259
278
  }
260
279
 
261
280
  // TODO: Refactoring the StreamingQueryResult to use Push-based execution should eliminate the need for this code
@@ -1,16 +1,70 @@
1
1
  #include "duckdb/parallel/pipeline_finish_event.hpp"
2
2
  #include "duckdb/execution/executor.hpp"
3
+ #include "duckdb/parallel/interrupt.hpp"
3
4
 
4
5
  namespace duckdb {
5
6
 
7
+ //! The PipelineFinishTask calls Finalize on the sink. Note that this is a single-threaded operation, but is executed
8
+ //! in a task to allow the Finalize call to block (e.g. for async I/O)
9
+ class PipelineFinishTask : public ExecutorTask {
10
+ public:
11
+ explicit PipelineFinishTask(Pipeline &pipeline_p, shared_ptr<Event> event_p)
12
+ : ExecutorTask(pipeline_p.executor), pipeline(pipeline_p), event(std::move(event_p)) {
13
+ }
14
+
15
+ Pipeline &pipeline;
16
+ shared_ptr<Event> event;
17
+
18
+ public:
19
+ TaskExecutionResult ExecuteTask(TaskExecutionMode mode) override {
20
+ auto sink = pipeline.GetSink();
21
+ InterruptState interrupt_state(shared_from_this());
22
+ OperatorSinkFinalizeInput finalize_input {*sink->sink_state, interrupt_state};
23
+
24
+ #ifdef DUCKDB_DEBUG_ASYNC_SINK_SOURCE
25
+ if (debug_blocked_count < debug_blocked_target_count) {
26
+ debug_blocked_count++;
27
+
28
+ auto &callback_state = interrupt_state;
29
+ std::thread rewake_thread([callback_state] {
30
+ std::this_thread::sleep_for(std::chrono::milliseconds(1));
31
+ callback_state.Callback();
32
+ });
33
+ rewake_thread.detach();
34
+
35
+ return TaskExecutionResult::TASK_BLOCKED;
36
+ }
37
+ #endif
38
+ auto sink_state = sink->Finalize(pipeline, *event, executor.context, finalize_input);
39
+
40
+ if (sink_state == SinkFinalizeType::BLOCKED) {
41
+ return TaskExecutionResult::TASK_BLOCKED;
42
+ }
43
+
44
+ sink->sink_state->state = sink_state;
45
+ event->FinishTask();
46
+ return TaskExecutionResult::TASK_FINISHED;
47
+ }
48
+
49
+ private:
50
+ #ifdef DUCKDB_DEBUG_ASYNC_SINK_SOURCE
51
+ //! Debugging state: number of times blocked
52
+ int debug_blocked_count = 0;
53
+ //! Number of times the Finalize will block before actually returning data
54
+ int debug_blocked_target_count = 1;
55
+ #endif
56
+ };
57
+
6
58
  PipelineFinishEvent::PipelineFinishEvent(shared_ptr<Pipeline> pipeline_p) : BasePipelineEvent(std::move(pipeline_p)) {
7
59
  }
8
60
 
9
61
  void PipelineFinishEvent::Schedule() {
62
+ vector<shared_ptr<Task>> tasks;
63
+ tasks.push_back(make_uniq<PipelineFinishTask>(*pipeline, shared_from_this()));
64
+ SetTasks(std::move(tasks));
10
65
  }
11
66
 
12
67
  void PipelineFinishEvent::FinishEvent() {
13
- pipeline->Finalize(*this);
14
68
  }
15
69
 
16
70
  } // namespace duckdb
@@ -21,15 +21,25 @@ unique_ptr<BoundTableRef> Binder::Bind(BaseTableRef &ref) {
21
21
  QueryErrorContext error_context(root_statement, ref.query_location);
22
22
  // CTEs and views are also referred to using BaseTableRefs, hence need to distinguish here
23
23
  // check if the table name refers to a CTE
24
- auto found_cte = FindCTE(ref.table_name, ref.table_name == alias);
24
+
25
+ // CTE name should never be qualified (i.e. schema_name should be empty)
26
+ optional_ptr<CommonTableExpressionInfo> found_cte = nullptr;
27
+ if (ref.schema_name.empty()) {
28
+ found_cte = FindCTE(ref.table_name, ref.table_name == alias);
29
+ }
30
+
25
31
  if (found_cte) {
26
32
  // Check if there is a CTE binding in the BindContext
27
33
  auto &cte = *found_cte;
28
34
  auto ctebinding = bind_context.GetCTEBinding(ref.table_name);
29
35
  if (!ctebinding) {
30
36
  if (CTEIsAlreadyBound(cte)) {
31
- throw BinderException("Circular reference to CTE \"%s\", use WITH RECURSIVE to use recursive CTEs",
32
- ref.table_name);
37
+ throw BinderException(
38
+ "Circular reference to CTE \"%s\", There are two possible solutions. \n1. use WITH RECURSIVE to "
39
+ "use recursive CTEs. \n2. If "
40
+ "you want to use the TABLE name \"%s\" the same as the CTE name, please explicitly add "
41
+ "\"SCHEMA\" before table name. You can try \"main.%s\" (main is the duckdb default schema)",
42
+ ref.table_name, ref.table_name, ref.table_name);
33
43
  }
34
44
  // Move CTE to subquery and bind recursively
35
45
  SubqueryRef subquery(unique_ptr_cast<SQLStatement, SelectStatement>(cte.query->Copy()));