duckdb 0.8.2-dev1862.0 → 0.8.2-dev1968.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/binding.gyp +9 -9
  2. package/package.json +1 -1
  3. package/src/duckdb/src/common/enum_util.cpp +28 -0
  4. package/src/duckdb/src/common/types/hugeint.cpp +40 -0
  5. package/src/duckdb/src/core_functions/function_list.cpp +1 -0
  6. package/src/duckdb/src/core_functions/scalar/string/to_base.cpp +66 -0
  7. package/src/duckdb/src/execution/operator/aggregate/physical_hash_aggregate.cpp +14 -11
  8. package/src/duckdb/src/execution/operator/aggregate/physical_perfecthash_aggregate.cpp +6 -4
  9. package/src/duckdb/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp +14 -12
  10. package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +6 -4
  11. package/src/duckdb/src/execution/operator/helper/physical_batch_collector.cpp +8 -6
  12. package/src/duckdb/src/execution/operator/helper/physical_explain_analyze.cpp +2 -2
  13. package/src/duckdb/src/execution/operator/helper/physical_limit.cpp +5 -3
  14. package/src/duckdb/src/execution/operator/helper/physical_materialized_collector.cpp +7 -5
  15. package/src/duckdb/src/execution/operator/helper/physical_vacuum.cpp +7 -5
  16. package/src/duckdb/src/execution/operator/join/physical_asof_join.cpp +5 -4
  17. package/src/duckdb/src/execution/operator/join/physical_blockwise_nl_join.cpp +2 -2
  18. package/src/duckdb/src/execution/operator/join/physical_delim_join.cpp +13 -6
  19. package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp +7 -5
  20. package/src/duckdb/src/execution/operator/join/physical_iejoin.cpp +7 -5
  21. package/src/duckdb/src/execution/operator/join/physical_nested_loop_join.cpp +7 -4
  22. package/src/duckdb/src/execution/operator/join/physical_piecewise_merge_join.cpp +8 -6
  23. package/src/duckdb/src/execution/operator/order/physical_order.cpp +7 -5
  24. package/src/duckdb/src/execution/operator/order/physical_top_n.cpp +7 -5
  25. package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp +8 -6
  26. package/src/duckdb/src/execution/operator/persistent/physical_batch_insert.cpp +8 -7
  27. package/src/duckdb/src/execution/operator/persistent/physical_copy_to_file.cpp +8 -6
  28. package/src/duckdb/src/execution/operator/persistent/physical_fixed_batch_copy.cpp +11 -9
  29. package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +10 -10
  30. package/src/duckdb/src/execution/operator/persistent/physical_update.cpp +4 -2
  31. package/src/duckdb/src/execution/operator/schema/physical_create_index.cpp +7 -6
  32. package/src/duckdb/src/execution/physical_operator.cpp +3 -2
  33. package/src/duckdb/src/function/table/version/pragma_version.cpp +2 -2
  34. package/src/duckdb/src/include/duckdb/common/bitpacking.hpp +70 -55
  35. package/src/duckdb/src/include/duckdb/common/enum_util.hpp +8 -0
  36. package/src/duckdb/src/include/duckdb/common/enums/operator_result_type.hpp +5 -1
  37. package/src/duckdb/src/include/duckdb/common/hugeint.hpp +15 -0
  38. package/src/duckdb/src/include/duckdb/common/limits.hpp +52 -149
  39. package/src/duckdb/src/include/duckdb/common/numeric_utils.hpp +48 -0
  40. package/src/duckdb/src/include/duckdb/core_functions/aggregate/distributive_functions.hpp +1 -1
  41. package/src/duckdb/src/include/duckdb/core_functions/scalar/operators_functions.hpp +1 -1
  42. package/src/duckdb/src/include/duckdb/core_functions/scalar/string_functions.hpp +9 -0
  43. package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_hash_aggregate.hpp +3 -3
  44. package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_perfecthash_aggregate.hpp +1 -1
  45. package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_ungrouped_aggregate.hpp +3 -3
  46. package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_window.hpp +2 -2
  47. package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_batch_collector.hpp +2 -2
  48. package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_explain_analyze.hpp +1 -1
  49. package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_limit.hpp +1 -1
  50. package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_materialized_collector.hpp +1 -1
  51. package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_vacuum.hpp +2 -2
  52. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_asof_join.hpp +2 -2
  53. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_blockwise_nl_join.hpp +1 -1
  54. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_delim_join.hpp +2 -2
  55. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_hash_join.hpp +2 -2
  56. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_iejoin.hpp +2 -2
  57. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_nested_loop_join.hpp +2 -2
  58. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_piecewise_merge_join.hpp +2 -2
  59. package/src/duckdb/src/include/duckdb/execution/operator/order/physical_order.hpp +2 -2
  60. package/src/duckdb/src/include/duckdb/execution/operator/order/physical_top_n.hpp +2 -2
  61. package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_batch_copy_to_file.hpp +2 -2
  62. package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_batch_insert.hpp +2 -2
  63. package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_copy_to_file.hpp +2 -2
  64. package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_fixed_batch_copy.hpp +2 -2
  65. package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_insert.hpp +2 -2
  66. package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_update.hpp +1 -1
  67. package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_create_index.hpp +2 -2
  68. package/src/duckdb/src/include/duckdb/execution/physical_operator.hpp +3 -3
  69. package/src/duckdb/src/include/duckdb/execution/physical_operator_states.hpp +11 -0
  70. package/src/duckdb/src/include/duckdb/main/extension_entries.hpp +3 -0
  71. package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp +0 -3
  72. package/src/duckdb/src/include/duckdb/parallel/pipeline_executor.hpp +3 -2
  73. package/src/duckdb/src/main/relation/join_relation.cpp +1 -1
  74. package/src/duckdb/src/parallel/pipeline.cpp +0 -17
  75. package/src/duckdb/src/parallel/pipeline_executor.cpp +26 -7
  76. package/src/duckdb/src/parallel/pipeline_finish_event.cpp +55 -1
  77. package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +13 -3
  78. package/src/duckdb/src/storage/compression/bitpacking.cpp +87 -63
  79. package/src/duckdb/src/storage/compression/bitpacking_hugeint.cpp +295 -0
  80. package/src/duckdb/ub_extension_icu_third_party_icu_i18n.cpp +6 -6
  81. package/src/duckdb/ub_src_core_functions_scalar_string.cpp +2 -0
  82. package/src/duckdb/ub_src_storage_compression.cpp +2 -0
@@ -0,0 +1,295 @@
1
+ #include "duckdb/common/bitpacking.hpp"
2
+
3
+ namespace duckdb {
4
+
5
+ //===--------------------------------------------------------------------===//
6
+ // Unpacking
7
+ //===--------------------------------------------------------------------===//
8
+
9
+ static void UnpackSingle(const uint32_t *__restrict &in, hugeint_t *__restrict out, uint16_t delta, uint16_t shr) {
10
+ if (delta + shr < 32) {
11
+ *out = ((static_cast<hugeint_t>(in[0])) >> shr) % (hugeint_t(1) << delta);
12
+ }
13
+
14
+ else if (delta + shr >= 32 && delta + shr < 64) {
15
+ *out = static_cast<hugeint_t>(in[0]) >> shr;
16
+ ++in;
17
+
18
+ if (delta + shr > 32) {
19
+ const uint16_t NEXT_SHR = shr + delta - 32;
20
+ *out |= static_cast<hugeint_t>((*in) % (1U << NEXT_SHR)) << (32 - shr);
21
+ }
22
+ }
23
+
24
+ else if (delta + shr >= 64 && delta + shr < 96) {
25
+ *out = static_cast<hugeint_t>(in[0]) >> shr;
26
+ *out |= static_cast<hugeint_t>(in[1]) << (32 - shr);
27
+ in += 2;
28
+
29
+ if (delta + shr > 64) {
30
+ const uint16_t NEXT_SHR = delta + shr - 64;
31
+ *out |= static_cast<hugeint_t>((*in) % (1U << NEXT_SHR)) << (64 - shr);
32
+ }
33
+ }
34
+
35
+ else if (delta + shr >= 96 && delta + shr < 128) {
36
+ *out = static_cast<hugeint_t>(in[0]) >> shr;
37
+ *out |= static_cast<hugeint_t>(in[1]) << (32 - shr);
38
+ *out |= static_cast<hugeint_t>(in[2]) << (64 - shr);
39
+ in += 3;
40
+
41
+ if (delta + shr > 96) {
42
+ const uint16_t NEXT_SHR = delta + shr - 96;
43
+ *out |= static_cast<hugeint_t>((*in) % (1U << NEXT_SHR)) << (96 - shr);
44
+ }
45
+ }
46
+
47
+ else if (delta + shr >= 128) {
48
+ *out = static_cast<hugeint_t>(in[0]) >> shr;
49
+ *out |= static_cast<hugeint_t>(in[1]) << (32 - shr);
50
+ *out |= static_cast<hugeint_t>(in[2]) << (64 - shr);
51
+ *out |= static_cast<hugeint_t>(in[3]) << (96 - shr);
52
+ in += 4;
53
+
54
+ if (delta + shr > 128) {
55
+ const uint16_t NEXT_SHR = delta + shr - 128;
56
+ *out |= static_cast<hugeint_t>((*in) % (1U << NEXT_SHR)) << (128 - shr);
57
+ }
58
+ }
59
+ }
60
+
61
+ static void UnpackLast(const uint32_t *__restrict &in, hugeint_t *__restrict out, uint16_t delta) {
62
+ const uint8_t LAST_IDX = 31;
63
+ const uint16_t SHIFT = (delta * 31) % 32;
64
+ out[LAST_IDX] = in[0] >> SHIFT;
65
+ if (delta > 32) {
66
+ out[LAST_IDX] |= static_cast<hugeint_t>(in[1]) << (32 - SHIFT);
67
+ }
68
+ if (delta > 64) {
69
+ out[LAST_IDX] |= static_cast<hugeint_t>(in[2]) << (64 - SHIFT);
70
+ }
71
+ if (delta > 96) {
72
+ out[LAST_IDX] |= static_cast<hugeint_t>(in[3]) << (96 - SHIFT);
73
+ }
74
+ }
75
+
76
+ // Unpacks for specific deltas
77
+ static void UnpackDelta0(const uint32_t *__restrict in, hugeint_t *__restrict out) {
78
+ for (uint8_t i = 0; i < 32; ++i) {
79
+ out[i] = 0;
80
+ }
81
+ }
82
+
83
+ static void UnpackDelta32(const uint32_t *__restrict in, hugeint_t *__restrict out) {
84
+ for (uint8_t k = 0; k < 32; ++k) {
85
+ out[k] = static_cast<hugeint_t>(in[k]);
86
+ }
87
+ }
88
+
89
+ static void UnpackDelta64(const uint32_t *__restrict in, hugeint_t *__restrict out) {
90
+ for (uint8_t i = 0; i < 32; ++i) {
91
+ const uint8_t OFFSET = i * 2;
92
+ out[i] = in[OFFSET];
93
+ out[i] |= static_cast<hugeint_t>(in[OFFSET + 1]) << 32;
94
+ }
95
+ }
96
+
97
+ static void UnpackDelta96(const uint32_t *__restrict in, hugeint_t *__restrict out) {
98
+ for (uint8_t i = 0; i < 32; ++i) {
99
+ const uint8_t OFFSET = i * 3;
100
+ out[i] = in[OFFSET];
101
+ out[i] |= static_cast<hugeint_t>(in[OFFSET + 1]) << 32;
102
+ out[i] |= static_cast<hugeint_t>(in[OFFSET + 2]) << 64;
103
+ }
104
+ }
105
+
106
+ static void UnpackDelta128(const uint32_t *__restrict in, hugeint_t *__restrict out) {
107
+ for (uint8_t i = 0; i < 32; ++i) {
108
+ const uint8_t OFFSET = i * 4;
109
+ out[i] = in[OFFSET];
110
+ out[i] |= static_cast<hugeint_t>(in[OFFSET + 1]) << 32;
111
+ out[i] |= static_cast<hugeint_t>(in[OFFSET + 2]) << 64;
112
+ out[i] |= static_cast<hugeint_t>(in[OFFSET + 3]) << 96;
113
+ }
114
+ }
115
+
116
+ //===--------------------------------------------------------------------===//
117
+ // Packing
118
+ //===--------------------------------------------------------------------===//
119
+
120
+ static void PackSingle(const hugeint_t in, uint32_t *__restrict &out, uint16_t delta, uint16_t shl, hugeint_t mask) {
121
+ if (delta + shl < 32) {
122
+
123
+ if (shl == 0) {
124
+ out[0] = static_cast<uint32_t>(in & mask);
125
+ } else {
126
+ out[0] |= static_cast<uint32_t>((in & mask) << shl);
127
+ }
128
+
129
+ } else if (delta + shl >= 32 && delta + shl < 64) {
130
+
131
+ if (shl == 0) {
132
+ out[0] = static_cast<uint32_t>(in & mask);
133
+ } else {
134
+ out[0] |= static_cast<uint32_t>((in & mask) << shl);
135
+ }
136
+ ++out;
137
+
138
+ if (delta + shl > 32) {
139
+ *out = static_cast<uint32_t>((in & mask) >> (32 - shl));
140
+ }
141
+ }
142
+
143
+ else if (delta + shl >= 64 && delta + shl < 96) {
144
+
145
+ if (shl == 0) {
146
+ out[0] = static_cast<uint32_t>(in & mask);
147
+ } else {
148
+ out[0] |= static_cast<uint32_t>(in << shl);
149
+ }
150
+
151
+ out[1] = static_cast<uint32_t>((in & mask) >> (32 - shl));
152
+ out += 2;
153
+
154
+ if (delta + shl > 64) {
155
+ *out = static_cast<uint32_t>((in & mask) >> (64 - shl));
156
+ }
157
+ }
158
+
159
+ else if (delta + shl >= 96 && delta + shl < 128) {
160
+ if (shl == 0) {
161
+ out[0] = static_cast<uint32_t>(in & mask);
162
+ } else {
163
+ out[0] |= static_cast<uint32_t>(in << shl);
164
+ }
165
+
166
+ out[1] = static_cast<uint32_t>((in & mask) >> (32 - shl));
167
+ out[2] = static_cast<uint32_t>((in & mask) >> (64 - shl));
168
+ out += 3;
169
+
170
+ if (delta + shl > 96) {
171
+ *out = static_cast<uint32_t>((in & mask) >> (96 - shl));
172
+ }
173
+ }
174
+
175
+ else if (delta + shl >= 128) {
176
+ // shl == 0 won't ever happen here considering a delta of 128 calls PackDelta128
177
+ out[0] |= static_cast<uint32_t>(in << shl);
178
+ out[1] = static_cast<uint32_t>((in & mask) >> (32 - shl));
179
+ out[2] = static_cast<uint32_t>((in & mask) >> (64 - shl));
180
+ out[3] = static_cast<uint32_t>((in & mask) >> (96 - shl));
181
+ out += 4;
182
+
183
+ if (delta + shl > 128) {
184
+ *out = static_cast<uint32_t>((in & mask) >> (128 - shl));
185
+ }
186
+ }
187
+ }
188
+
189
+ static void PackLast(const hugeint_t *__restrict in, uint32_t *__restrict out, uint16_t delta) {
190
+ const uint8_t LAST_IDX = 31;
191
+ const uint16_t SHIFT = (delta * 31) % 32;
192
+ out[0] |= static_cast<uint32_t>(in[LAST_IDX] << SHIFT);
193
+ if (delta > 32) {
194
+ out[1] = static_cast<uint32_t>(in[LAST_IDX] >> (32 - SHIFT));
195
+ }
196
+ if (delta > 64) {
197
+ out[2] = static_cast<uint32_t>(in[LAST_IDX] >> (64 - SHIFT));
198
+ }
199
+ if (delta > 96) {
200
+ out[3] = static_cast<uint32_t>(in[LAST_IDX] >> (96 - SHIFT));
201
+ }
202
+ }
203
+
204
+ // Packs for specific deltas
205
+ static void PackDelta32(const hugeint_t *__restrict in, uint32_t *__restrict out) {
206
+ for (uint8_t i = 0; i < 32; ++i) {
207
+ out[i] = static_cast<uint32_t>(in[i]);
208
+ }
209
+ }
210
+
211
+ static void PackDelta64(const hugeint_t *__restrict in, uint32_t *__restrict out) {
212
+ for (uint8_t i = 0; i < 32; ++i) {
213
+ const uint8_t OFFSET = 2 * i;
214
+ out[OFFSET] = static_cast<uint32_t>(in[i]);
215
+ out[OFFSET + 1] = static_cast<uint32_t>(in[i] >> 32);
216
+ }
217
+ }
218
+
219
+ static void PackDelta96(const hugeint_t *__restrict in, uint32_t *__restrict out) {
220
+ for (uint8_t i = 0; i < 32; ++i) {
221
+ const uint8_t OFFSET = 3 * i;
222
+ out[OFFSET] = static_cast<uint32_t>(in[i]);
223
+ out[OFFSET + 1] = static_cast<uint32_t>(in[i] >> 32);
224
+ out[OFFSET + 2] = static_cast<uint32_t>(in[i] >> 64);
225
+ }
226
+ }
227
+
228
+ static void PackDelta128(const hugeint_t *__restrict in, uint32_t *__restrict out) {
229
+ for (uint8_t i = 0; i < 32; ++i) {
230
+ const uint8_t OFFSET = 4 * i;
231
+ out[OFFSET] = static_cast<uint32_t>(in[i]);
232
+ out[OFFSET + 1] = static_cast<uint32_t>(in[i] >> 32);
233
+ out[OFFSET + 2] = static_cast<uint32_t>(in[i] >> 64);
234
+ out[OFFSET + 3] = static_cast<uint32_t>(in[i] >> 96);
235
+ }
236
+ }
237
+
238
+ //===--------------------------------------------------------------------===//
239
+ // HugeIntPacker
240
+ //===--------------------------------------------------------------------===//
241
+
242
+ void HugeIntPacker::Pack(const hugeint_t *__restrict in, uint32_t *__restrict out, bitpacking_width_t width) {
243
+ D_ASSERT(width <= 128);
244
+ switch (width) {
245
+ case 0:
246
+ break;
247
+ case 32:
248
+ PackDelta32(in, out);
249
+ break;
250
+ case 64:
251
+ PackDelta64(in, out);
252
+ break;
253
+ case 96:
254
+ PackDelta96(in, out);
255
+ break;
256
+ case 128:
257
+ PackDelta128(in, out);
258
+ break;
259
+ default:
260
+ for (idx_t oindex = 0; oindex < BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE - 1; ++oindex) {
261
+ PackSingle(in[oindex], out, width, (width * oindex) % BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE,
262
+ (hugeint_t(1) << width) - 1);
263
+ }
264
+ PackLast(in, out, width);
265
+ }
266
+ }
267
+
268
+ void HugeIntPacker::Unpack(const uint32_t *__restrict in, hugeint_t *__restrict out, bitpacking_width_t width) {
269
+ D_ASSERT(width <= 128);
270
+ switch (width) {
271
+ case 0:
272
+ UnpackDelta0(in, out);
273
+ break;
274
+ case 32:
275
+ UnpackDelta32(in, out);
276
+ break;
277
+ case 64:
278
+ UnpackDelta64(in, out);
279
+ break;
280
+ case 96:
281
+ UnpackDelta96(in, out);
282
+ break;
283
+ case 128:
284
+ UnpackDelta128(in, out);
285
+ break;
286
+ default:
287
+ for (idx_t oindex = 0; oindex < BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE - 1; ++oindex) {
288
+ UnpackSingle(in, out + oindex, width,
289
+ (width * oindex) % BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE);
290
+ }
291
+ UnpackLast(in, out, width);
292
+ }
293
+ }
294
+
295
+ } // namespace duckdb
@@ -348,17 +348,17 @@
348
348
 
349
349
  #include "extension/icu/third_party/icu/i18n/wintzimpl.cpp"
350
350
 
351
- #include "extension/icu/third_party/icu/i18n/double-conversion-strtod.cpp"
352
-
353
- #include "extension/icu/third_party/icu/i18n/double-conversion-fast-dtoa.cpp"
351
+ #include "extension/icu/third_party/icu/i18n/double-conversion-bignum.cpp"
354
352
 
355
- #include "extension/icu/third_party/icu/i18n/double-conversion-string-to-double.cpp"
353
+ #include "extension/icu/third_party/icu/i18n/double-conversion-cached-powers.cpp"
356
354
 
357
- #include "extension/icu/third_party/icu/i18n/double-conversion-bignum.cpp"
355
+ #include "extension/icu/third_party/icu/i18n/double-conversion-strtod.cpp"
358
356
 
359
357
  #include "extension/icu/third_party/icu/i18n/double-conversion-double-to-string.cpp"
360
358
 
359
+ #include "extension/icu/third_party/icu/i18n/double-conversion-string-to-double.cpp"
360
+
361
361
  #include "extension/icu/third_party/icu/i18n/double-conversion-bignum-dtoa.cpp"
362
362
 
363
- #include "extension/icu/third_party/icu/i18n/double-conversion-cached-powers.cpp"
363
+ #include "extension/icu/third_party/icu/i18n/double-conversion-fast-dtoa.cpp"
364
364
 
@@ -38,6 +38,8 @@
38
38
 
39
39
  #include "src/core_functions/scalar/string/string_split.cpp"
40
40
 
41
+ #include "src/core_functions/scalar/string/to_base.cpp"
42
+
41
43
  #include "src/core_functions/scalar/string/translate.cpp"
42
44
 
43
45
  #include "src/core_functions/scalar/string/trim.cpp"
@@ -14,6 +14,8 @@
14
14
 
15
15
  #include "src/storage/compression/bitpacking.cpp"
16
16
 
17
+ #include "src/storage/compression/bitpacking_hugeint.cpp"
18
+
17
19
  #include "src/storage/compression/patas.cpp"
18
20
 
19
21
  #include "src/storage/compression/fsst.cpp"