duckdb 1.1.3 → 1.1.4-dev9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1747) hide show
  1. package/.github/workflows/NodeJS.yml +5 -54
  2. package/binding.gyp +73 -52
  3. package/package.json +1 -1
  4. package/src/duckdb/{src → extension}/core_functions/aggregate/algebraic/avg.cpp +2 -2
  5. package/src/duckdb/{src → extension}/core_functions/aggregate/algebraic/corr.cpp +4 -4
  6. package/src/duckdb/{src → extension}/core_functions/aggregate/algebraic/covar.cpp +2 -2
  7. package/src/duckdb/{src → extension}/core_functions/aggregate/algebraic/stddev.cpp +2 -2
  8. package/src/duckdb/{src → extension}/core_functions/aggregate/distributive/approx_count.cpp +1 -1
  9. package/src/duckdb/{src → extension}/core_functions/aggregate/distributive/arg_min_max.cpp +66 -18
  10. package/src/duckdb/{src → extension}/core_functions/aggregate/distributive/bitagg.cpp +1 -1
  11. package/src/duckdb/{src → extension}/core_functions/aggregate/distributive/bitstring_agg.cpp +5 -7
  12. package/src/duckdb/{src → extension}/core_functions/aggregate/distributive/bool.cpp +3 -1
  13. package/src/duckdb/{src → extension}/core_functions/aggregate/distributive/kurtosis.cpp +1 -1
  14. package/src/duckdb/{src → extension}/core_functions/aggregate/distributive/product.cpp +1 -1
  15. package/src/duckdb/{src → extension}/core_functions/aggregate/distributive/skew.cpp +2 -2
  16. package/src/duckdb/{src → extension}/core_functions/aggregate/distributive/string_agg.cpp +1 -1
  17. package/src/duckdb/{src → extension}/core_functions/aggregate/distributive/sum.cpp +13 -2
  18. package/src/duckdb/{src → extension}/core_functions/aggregate/holistic/approx_top_k.cpp +3 -3
  19. package/src/duckdb/{src → extension}/core_functions/aggregate/holistic/approximate_quantile.cpp +51 -15
  20. package/src/duckdb/{src → extension}/core_functions/aggregate/holistic/mad.cpp +25 -10
  21. package/src/duckdb/{src → extension}/core_functions/aggregate/holistic/mode.cpp +215 -71
  22. package/src/duckdb/{src → extension}/core_functions/aggregate/holistic/quantile.cpp +58 -31
  23. package/src/duckdb/{src → extension}/core_functions/aggregate/holistic/reservoir_quantile.cpp +2 -2
  24. package/src/duckdb/{src → extension}/core_functions/aggregate/nested/binned_histogram.cpp +9 -4
  25. package/src/duckdb/{src → extension}/core_functions/aggregate/nested/histogram.cpp +4 -2
  26. package/src/duckdb/{src → extension}/core_functions/aggregate/nested/list.cpp +1 -1
  27. package/src/duckdb/{src → extension}/core_functions/aggregate/regression/regr_avg.cpp +1 -1
  28. package/src/duckdb/{src → extension}/core_functions/aggregate/regression/regr_count.cpp +2 -2
  29. package/src/duckdb/{src → extension}/core_functions/aggregate/regression/regr_intercept.cpp +6 -2
  30. package/src/duckdb/{src → extension}/core_functions/aggregate/regression/regr_r2.cpp +2 -2
  31. package/src/duckdb/{src → extension}/core_functions/aggregate/regression/regr_slope.cpp +2 -2
  32. package/src/duckdb/{src → extension}/core_functions/aggregate/regression/regr_sxx_syy.cpp +2 -2
  33. package/src/duckdb/{src → extension}/core_functions/aggregate/regression/regr_sxy.cpp +3 -3
  34. package/src/duckdb/extension/core_functions/core_functions_extension.cpp +85 -0
  35. package/src/duckdb/{src → extension}/core_functions/function_list.cpp +30 -51
  36. package/src/duckdb/{src/include/duckdb → extension/core_functions/include}/core_functions/aggregate/algebraic/corr.hpp +3 -7
  37. package/src/duckdb/{src/include/duckdb → extension/core_functions/include}/core_functions/aggregate/algebraic_functions.hpp +1 -1
  38. package/src/duckdb/{src/include/duckdb → extension/core_functions/include}/core_functions/aggregate/distributive_functions.hpp +16 -21
  39. package/src/duckdb/{src/include/duckdb → extension/core_functions/include}/core_functions/aggregate/histogram_helpers.hpp +1 -1
  40. package/src/duckdb/{src/include/duckdb → extension/core_functions/include}/core_functions/aggregate/holistic_functions.hpp +1 -1
  41. package/src/duckdb/{src/include/duckdb → extension/core_functions/include}/core_functions/aggregate/nested_functions.hpp +1 -1
  42. package/src/duckdb/{src/include/duckdb → extension/core_functions/include}/core_functions/aggregate/quantile_helpers.hpp +2 -2
  43. package/src/duckdb/{src/include/duckdb → extension/core_functions/include}/core_functions/aggregate/quantile_sort_tree.hpp +140 -58
  44. package/src/duckdb/{src/include/duckdb → extension/core_functions/include}/core_functions/aggregate/quantile_state.hpp +50 -43
  45. package/src/duckdb/{src/include/duckdb → extension/core_functions/include}/core_functions/aggregate/regression/regr_count.hpp +2 -2
  46. package/src/duckdb/{src/include/duckdb → extension/core_functions/include}/core_functions/aggregate/regression/regr_slope.hpp +3 -7
  47. package/src/duckdb/{src/include/duckdb → extension/core_functions/include}/core_functions/aggregate/regression_functions.hpp +1 -1
  48. package/src/duckdb/{src/include/duckdb → extension/core_functions/include}/core_functions/scalar/array_functions.hpp +1 -1
  49. package/src/duckdb/{src/include/duckdb → extension/core_functions/include}/core_functions/scalar/bit_functions.hpp +1 -1
  50. package/src/duckdb/{src/include/duckdb → extension/core_functions/include}/core_functions/scalar/blob_functions.hpp +1 -10
  51. package/src/duckdb/{src/include/duckdb → extension/core_functions/include}/core_functions/scalar/date_functions.hpp +22 -55
  52. package/src/duckdb/{src/include/duckdb → extension/core_functions/include}/core_functions/scalar/debug_functions.hpp +1 -1
  53. package/src/duckdb/{src/include/duckdb → extension/core_functions/include}/core_functions/scalar/enum_functions.hpp +1 -1
  54. package/src/duckdb/{src/include/duckdb → extension/core_functions/include}/core_functions/scalar/generic_functions.hpp +1 -10
  55. package/src/duckdb/{src/include/duckdb → extension/core_functions/include}/core_functions/scalar/list_functions.hpp +4 -4
  56. package/src/duckdb/{src/include/duckdb → extension/core_functions/include}/core_functions/scalar/map_functions.hpp +1 -10
  57. package/src/duckdb/{src/include/duckdb → extension/core_functions/include}/core_functions/scalar/math_functions.hpp +1 -1
  58. package/src/duckdb/{src/include/duckdb → extension/core_functions/include}/core_functions/scalar/operators_functions.hpp +1 -1
  59. package/src/duckdb/{src/include/duckdb → extension/core_functions/include}/core_functions/scalar/random_functions.hpp +1 -1
  60. package/src/duckdb/{src/include/duckdb → extension/core_functions/include}/core_functions/scalar/string_functions.hpp +10 -103
  61. package/src/duckdb/{src/include/duckdb → extension/core_functions/include}/core_functions/scalar/struct_functions.hpp +1 -19
  62. package/src/duckdb/{src/include/duckdb → extension/core_functions/include}/core_functions/scalar/union_functions.hpp +1 -1
  63. package/src/duckdb/extension/core_functions/include/core_functions_extension.hpp +22 -0
  64. package/src/duckdb/{src → extension}/core_functions/lambda_functions.cpp +1 -1
  65. package/src/duckdb/{src → extension}/core_functions/scalar/array/array_functions.cpp +11 -4
  66. package/src/duckdb/{src → extension}/core_functions/scalar/array/array_value.cpp +2 -2
  67. package/src/duckdb/{src → extension}/core_functions/scalar/bit/bitstring.cpp +12 -5
  68. package/src/duckdb/{src → extension}/core_functions/scalar/blob/base64.cpp +4 -2
  69. package/src/duckdb/{src → extension}/core_functions/scalar/blob/encode.cpp +4 -2
  70. package/src/duckdb/{src → extension}/core_functions/scalar/date/age.cpp +9 -3
  71. package/src/duckdb/extension/core_functions/scalar/date/current.cpp +29 -0
  72. package/src/duckdb/{src → extension}/core_functions/scalar/date/date_diff.cpp +1 -1
  73. package/src/duckdb/{src → extension}/core_functions/scalar/date/date_part.cpp +42 -9
  74. package/src/duckdb/{src → extension}/core_functions/scalar/date/date_sub.cpp +1 -1
  75. package/src/duckdb/{src → extension}/core_functions/scalar/date/date_trunc.cpp +4 -1
  76. package/src/duckdb/{src → extension}/core_functions/scalar/date/epoch.cpp +19 -3
  77. package/src/duckdb/{src → extension}/core_functions/scalar/date/make_date.cpp +40 -5
  78. package/src/duckdb/{src → extension}/core_functions/scalar/date/time_bucket.cpp +4 -1
  79. package/src/duckdb/{src → extension}/core_functions/scalar/date/to_interval.cpp +54 -28
  80. package/src/duckdb/{src → extension}/core_functions/scalar/debug/vector_type.cpp +1 -1
  81. package/src/duckdb/{src → extension}/core_functions/scalar/enum/enum_functions.cpp +2 -7
  82. package/src/duckdb/{src → extension}/core_functions/scalar/generic/alias.cpp +2 -2
  83. package/src/duckdb/{src/function → extension/core_functions}/scalar/generic/binning.cpp +4 -3
  84. package/src/duckdb/{src → extension}/core_functions/scalar/generic/can_implicitly_cast.cpp +1 -1
  85. package/src/duckdb/{src → extension}/core_functions/scalar/generic/current_setting.cpp +1 -1
  86. package/src/duckdb/{src → extension}/core_functions/scalar/generic/hash.cpp +1 -1
  87. package/src/duckdb/{src → extension}/core_functions/scalar/generic/least.cpp +30 -10
  88. package/src/duckdb/{src → extension}/core_functions/scalar/generic/stats.cpp +1 -1
  89. package/src/duckdb/{src → extension}/core_functions/scalar/generic/system_functions.cpp +1 -1
  90. package/src/duckdb/{src → extension}/core_functions/scalar/generic/typeof.cpp +1 -1
  91. package/src/duckdb/{src → extension}/core_functions/scalar/list/array_slice.cpp +93 -88
  92. package/src/duckdb/{src → extension}/core_functions/scalar/list/flatten.cpp +1 -1
  93. package/src/duckdb/{src → extension}/core_functions/scalar/list/list_aggregates.cpp +7 -3
  94. package/src/duckdb/{src → extension}/core_functions/scalar/list/list_distance.cpp +8 -2
  95. package/src/duckdb/{src → extension}/core_functions/scalar/list/list_filter.cpp +3 -3
  96. package/src/duckdb/{src → extension}/core_functions/scalar/list/list_has_any_or_all.cpp +3 -3
  97. package/src/duckdb/{src → extension}/core_functions/scalar/list/list_reduce.cpp +5 -5
  98. package/src/duckdb/{src → extension}/core_functions/scalar/list/list_sort.cpp +1 -1
  99. package/src/duckdb/{src → extension}/core_functions/scalar/list/list_transform.cpp +3 -3
  100. package/src/duckdb/{src → extension}/core_functions/scalar/list/list_value.cpp +3 -3
  101. package/src/duckdb/{src → extension}/core_functions/scalar/list/range.cpp +7 -1
  102. package/src/duckdb/{src → extension}/core_functions/scalar/map/cardinality.cpp +1 -1
  103. package/src/duckdb/{src → extension}/core_functions/scalar/map/map.cpp +5 -4
  104. package/src/duckdb/{src → extension}/core_functions/scalar/map/map_concat.cpp +1 -1
  105. package/src/duckdb/{src → extension}/core_functions/scalar/map/map_entries.cpp +1 -1
  106. package/src/duckdb/{src → extension}/core_functions/scalar/map/map_extract.cpp +13 -25
  107. package/src/duckdb/{src → extension}/core_functions/scalar/map/map_from_entries.cpp +2 -1
  108. package/src/duckdb/{src → extension}/core_functions/scalar/map/map_keys_values.cpp +11 -9
  109. package/src/duckdb/{src → extension}/core_functions/scalar/math/numeric.cpp +83 -37
  110. package/src/duckdb/{src → extension}/core_functions/scalar/operators/bitwise.cpp +19 -1
  111. package/src/duckdb/{src → extension}/core_functions/scalar/random/random.cpp +4 -3
  112. package/src/duckdb/{src → extension}/core_functions/scalar/random/setseed.cpp +2 -1
  113. package/src/duckdb/{src → extension}/core_functions/scalar/string/ascii.cpp +1 -1
  114. package/src/duckdb/{src → extension}/core_functions/scalar/string/bar.cpp +6 -4
  115. package/src/duckdb/{src → extension}/core_functions/scalar/string/chr.cpp +1 -1
  116. package/src/duckdb/{src → extension}/core_functions/scalar/string/damerau_levenshtein.cpp +1 -1
  117. package/src/duckdb/{src → extension}/core_functions/scalar/string/format_bytes.cpp +1 -1
  118. package/src/duckdb/{src → extension}/core_functions/scalar/string/hamming.cpp +1 -1
  119. package/src/duckdb/{src → extension}/core_functions/scalar/string/hex.cpp +7 -3
  120. package/src/duckdb/{src → extension}/core_functions/scalar/string/instr.cpp +4 -4
  121. package/src/duckdb/{src → extension}/core_functions/scalar/string/jaccard.cpp +1 -1
  122. package/src/duckdb/extension/core_functions/scalar/string/jaro_winkler.cpp +112 -0
  123. package/src/duckdb/{src → extension}/core_functions/scalar/string/left_right.cpp +6 -6
  124. package/src/duckdb/{src → extension}/core_functions/scalar/string/levenshtein.cpp +1 -1
  125. package/src/duckdb/{src → extension}/core_functions/scalar/string/pad.cpp +9 -5
  126. package/src/duckdb/{src → extension}/core_functions/scalar/string/parse_path.cpp +4 -4
  127. package/src/duckdb/{src → extension}/core_functions/scalar/string/printf.cpp +3 -1
  128. package/src/duckdb/{src → extension}/core_functions/scalar/string/repeat.cpp +4 -1
  129. package/src/duckdb/{src → extension}/core_functions/scalar/string/replace.cpp +1 -1
  130. package/src/duckdb/{src → extension}/core_functions/scalar/string/reverse.cpp +1 -1
  131. package/src/duckdb/{src → extension}/core_functions/scalar/string/starts_with.cpp +5 -3
  132. package/src/duckdb/{src → extension}/core_functions/scalar/string/to_base.cpp +1 -1
  133. package/src/duckdb/{src → extension}/core_functions/scalar/string/translate.cpp +1 -1
  134. package/src/duckdb/{src → extension}/core_functions/scalar/string/trim.cpp +1 -1
  135. package/src/duckdb/{src → extension}/core_functions/scalar/string/unicode.cpp +1 -1
  136. package/src/duckdb/{src → extension}/core_functions/scalar/string/url_encode.cpp +1 -1
  137. package/src/duckdb/{src → extension}/core_functions/scalar/struct/struct_insert.cpp +25 -31
  138. package/src/duckdb/{src → extension}/core_functions/scalar/union/union_extract.cpp +1 -1
  139. package/src/duckdb/{src → extension}/core_functions/scalar/union/union_tag.cpp +1 -1
  140. package/src/duckdb/{src → extension}/core_functions/scalar/union/union_value.cpp +3 -3
  141. package/src/duckdb/extension/icu/icu-dateadd.cpp +16 -11
  142. package/src/duckdb/extension/icu/icu-datefunc.cpp +2 -2
  143. package/src/duckdb/extension/icu/icu-datepart.cpp +8 -5
  144. package/src/duckdb/extension/icu/icu-datesub.cpp +2 -2
  145. package/src/duckdb/extension/icu/icu-datetrunc.cpp +8 -1
  146. package/src/duckdb/extension/icu/icu-list-range.cpp +2 -2
  147. package/src/duckdb/extension/icu/icu-makedate.cpp +18 -7
  148. package/src/duckdb/extension/icu/icu-strptime.cpp +9 -3
  149. package/src/duckdb/extension/icu/icu-table-range.cpp +2 -2
  150. package/src/duckdb/extension/icu/icu-timebucket.cpp +4 -1
  151. package/src/duckdb/extension/icu/icu-timezone.cpp +67 -1
  152. package/src/duckdb/extension/icu/icu_extension.cpp +60 -5
  153. package/src/duckdb/extension/icu/include/icu-datefunc.hpp +2 -1
  154. package/src/duckdb/extension/icu/third_party/icu/common/bytestriebuilder.cpp +1 -1
  155. package/src/duckdb/extension/icu/third_party/icu/common/dtintrv.cpp +1 -1
  156. package/src/duckdb/extension/icu/third_party/icu/common/filteredbrk.cpp +1 -1
  157. package/src/duckdb/extension/icu/third_party/icu/common/locid.cpp +1 -1
  158. package/src/duckdb/extension/icu/third_party/icu/common/lsr.cpp +1 -1
  159. package/src/duckdb/extension/icu/third_party/icu/common/lsr.h +2 -2
  160. package/src/duckdb/extension/icu/third_party/icu/common/messagepattern.cpp +2 -2
  161. package/src/duckdb/extension/icu/third_party/icu/common/normlzr.cpp +1 -1
  162. package/src/duckdb/extension/icu/third_party/icu/common/rbbinode.h +1 -1
  163. package/src/duckdb/extension/icu/third_party/icu/common/schriter.cpp +1 -1
  164. package/src/duckdb/extension/icu/third_party/icu/common/stringtriebuilder.cpp +8 -8
  165. package/src/duckdb/extension/icu/third_party/icu/common/ucharstriebuilder.cpp +1 -1
  166. package/src/duckdb/extension/icu/third_party/icu/common/uchriter.cpp +1 -1
  167. package/src/duckdb/extension/icu/third_party/icu/common/unicode/brkiter.h +2 -2
  168. package/src/duckdb/extension/icu/third_party/icu/common/unicode/bytestriebuilder.h +1 -1
  169. package/src/duckdb/extension/icu/third_party/icu/common/unicode/chariter.h +3 -3
  170. package/src/duckdb/extension/icu/third_party/icu/common/unicode/dtintrv.h +3 -3
  171. package/src/duckdb/extension/icu/third_party/icu/common/unicode/locid.h +3 -3
  172. package/src/duckdb/extension/icu/third_party/icu/common/unicode/messagepattern.h +4 -4
  173. package/src/duckdb/extension/icu/third_party/icu/common/unicode/normlzr.h +3 -3
  174. package/src/duckdb/extension/icu/third_party/icu/common/unicode/parsepos.h +4 -4
  175. package/src/duckdb/extension/icu/third_party/icu/common/unicode/rbbi.h +2 -2
  176. package/src/duckdb/extension/icu/third_party/icu/common/unicode/schriter.h +1 -1
  177. package/src/duckdb/extension/icu/third_party/icu/common/unicode/strenum.h +2 -2
  178. package/src/duckdb/extension/icu/third_party/icu/common/unicode/stringpiece.h +1 -1
  179. package/src/duckdb/extension/icu/third_party/icu/common/unicode/stringtriebuilder.h +9 -9
  180. package/src/duckdb/extension/icu/third_party/icu/common/unicode/ucharstriebuilder.h +1 -1
  181. package/src/duckdb/extension/icu/third_party/icu/common/unicode/uchriter.h +1 -1
  182. package/src/duckdb/extension/icu/third_party/icu/common/unicode/uniset.h +3 -3
  183. package/src/duckdb/extension/icu/third_party/icu/common/unicode/unistr.h +12 -12
  184. package/src/duckdb/extension/icu/third_party/icu/common/unicode/uobject.h +2 -2
  185. package/src/duckdb/extension/icu/third_party/icu/common/unifiedcache.h +4 -4
  186. package/src/duckdb/extension/icu/third_party/icu/common/uniset.cpp +1 -1
  187. package/src/duckdb/extension/icu/third_party/icu/common/ustr_titlecase_brkiter.cpp +1 -1
  188. package/src/duckdb/extension/icu/third_party/icu/common/ustrenum.cpp +2 -2
  189. package/src/duckdb/extension/icu/third_party/icu/common/uvector.cpp +1 -1
  190. package/src/duckdb/extension/icu/third_party/icu/common/uvector.h +3 -3
  191. package/src/duckdb/extension/icu/third_party/icu/common/uvectr32.cpp +1 -1
  192. package/src/duckdb/extension/icu/third_party/icu/common/uvectr32.h +3 -3
  193. package/src/duckdb/extension/icu/third_party/icu/common/uvectr64.cpp +1 -1
  194. package/src/duckdb/extension/icu/third_party/icu/common/uvectr64.h +3 -3
  195. package/src/duckdb/extension/icu/third_party/icu/i18n/alphaindex.cpp +2 -2
  196. package/src/duckdb/extension/icu/third_party/icu/i18n/calendar.cpp +1 -1
  197. package/src/duckdb/extension/icu/third_party/icu/i18n/choicfmt.cpp +1 -1
  198. package/src/duckdb/extension/icu/third_party/icu/i18n/coleitr.cpp +2 -2
  199. package/src/duckdb/extension/icu/third_party/icu/i18n/coll.cpp +2 -2
  200. package/src/duckdb/extension/icu/third_party/icu/i18n/collationiterator.cpp +1 -1
  201. package/src/duckdb/extension/icu/third_party/icu/i18n/collationiterator.h +2 -2
  202. package/src/duckdb/extension/icu/third_party/icu/i18n/collationsettings.cpp +1 -1
  203. package/src/duckdb/extension/icu/third_party/icu/i18n/collationsettings.h +2 -2
  204. package/src/duckdb/extension/icu/third_party/icu/i18n/currpinf.cpp +1 -1
  205. package/src/duckdb/extension/icu/third_party/icu/i18n/datefmt.cpp +2 -2
  206. package/src/duckdb/extension/icu/third_party/icu/i18n/dcfmtsym.cpp +1 -1
  207. package/src/duckdb/extension/icu/third_party/icu/i18n/decimfmt.cpp +1 -1
  208. package/src/duckdb/extension/icu/third_party/icu/i18n/dtfmtsym.cpp +1 -1
  209. package/src/duckdb/extension/icu/third_party/icu/i18n/dtitvfmt.cpp +1 -1
  210. package/src/duckdb/extension/icu/third_party/icu/i18n/dtitvinf.cpp +1 -1
  211. package/src/duckdb/extension/icu/third_party/icu/i18n/dtptngen.cpp +2 -2
  212. package/src/duckdb/extension/icu/third_party/icu/i18n/dtptngen_impl.h +4 -4
  213. package/src/duckdb/extension/icu/third_party/icu/i18n/dtrule.cpp +2 -2
  214. package/src/duckdb/extension/icu/third_party/icu/i18n/fmtable.cpp +1 -1
  215. package/src/duckdb/extension/icu/third_party/icu/i18n/format.cpp +1 -1
  216. package/src/duckdb/extension/icu/third_party/icu/i18n/fpositer.cpp +1 -1
  217. package/src/duckdb/extension/icu/third_party/icu/i18n/measfmt.cpp +1 -1
  218. package/src/duckdb/extension/icu/third_party/icu/i18n/measunit.cpp +1 -1
  219. package/src/duckdb/extension/icu/third_party/icu/i18n/measure.cpp +1 -1
  220. package/src/duckdb/extension/icu/third_party/icu/i18n/msgfmt.cpp +2 -2
  221. package/src/duckdb/extension/icu/third_party/icu/i18n/nfrs.cpp +1 -1
  222. package/src/duckdb/extension/icu/third_party/icu/i18n/nfrs.h +2 -2
  223. package/src/duckdb/extension/icu/third_party/icu/i18n/nfrule.cpp +1 -1
  224. package/src/duckdb/extension/icu/third_party/icu/i18n/nfrule.h +2 -2
  225. package/src/duckdb/extension/icu/third_party/icu/i18n/nfsubs.cpp +9 -9
  226. package/src/duckdb/extension/icu/third_party/icu/i18n/nfsubs.h +2 -2
  227. package/src/duckdb/extension/icu/third_party/icu/i18n/number_asformat.cpp +1 -1
  228. package/src/duckdb/extension/icu/third_party/icu/i18n/number_asformat.h +1 -1
  229. package/src/duckdb/extension/icu/third_party/icu/i18n/numfmt.cpp +1 -1
  230. package/src/duckdb/extension/icu/third_party/icu/i18n/olsontz.cpp +1 -1
  231. package/src/duckdb/extension/icu/third_party/icu/i18n/olsontz.h +1 -1
  232. package/src/duckdb/extension/icu/third_party/icu/i18n/plurfmt.cpp +2 -2
  233. package/src/duckdb/extension/icu/third_party/icu/i18n/plurrule.cpp +1 -1
  234. package/src/duckdb/extension/icu/third_party/icu/i18n/rbnf.cpp +4 -4
  235. package/src/duckdb/extension/icu/third_party/icu/i18n/rbtz.cpp +2 -2
  236. package/src/duckdb/extension/icu/third_party/icu/i18n/region.cpp +2 -2
  237. package/src/duckdb/extension/icu/third_party/icu/i18n/reldtfmt.cpp +1 -1
  238. package/src/duckdb/extension/icu/third_party/icu/i18n/reldtfmt.h +1 -1
  239. package/src/duckdb/extension/icu/third_party/icu/i18n/rulebasedcollator.cpp +1 -1
  240. package/src/duckdb/extension/icu/third_party/icu/i18n/selfmt.cpp +2 -2
  241. package/src/duckdb/extension/icu/third_party/icu/i18n/simpletz.cpp +1 -1
  242. package/src/duckdb/extension/icu/third_party/icu/i18n/smpdtfmt.cpp +1 -1
  243. package/src/duckdb/extension/icu/third_party/icu/i18n/sortkey.cpp +1 -1
  244. package/src/duckdb/extension/icu/third_party/icu/i18n/timezone.cpp +1 -1
  245. package/src/duckdb/extension/icu/third_party/icu/i18n/tmutamt.cpp +1 -1
  246. package/src/duckdb/extension/icu/third_party/icu/i18n/tzfmt.cpp +1 -1
  247. package/src/duckdb/extension/icu/third_party/icu/i18n/tzgnames.cpp +1 -1
  248. package/src/duckdb/extension/icu/third_party/icu/i18n/tzgnames.h +2 -2
  249. package/src/duckdb/extension/icu/third_party/icu/i18n/tznames.cpp +3 -3
  250. package/src/duckdb/extension/icu/third_party/icu/i18n/tznames_impl.cpp +2 -2
  251. package/src/duckdb/extension/icu/third_party/icu/i18n/tznames_impl.h +2 -2
  252. package/src/duckdb/extension/icu/third_party/icu/i18n/tzrule.cpp +8 -8
  253. package/src/duckdb/extension/icu/third_party/icu/i18n/tztrans.cpp +2 -2
  254. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/alphaindex.h +2 -2
  255. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/calendar.h +2 -2
  256. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/choicfmt.h +1 -1
  257. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/coleitr.h +2 -2
  258. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/coll.h +2 -2
  259. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/currpinf.h +3 -3
  260. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/datefmt.h +1 -1
  261. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/dcfmtsym.h +2 -2
  262. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/decimfmt.h +1 -1
  263. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/dtfmtsym.h +2 -2
  264. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/dtitvfmt.h +3 -3
  265. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/dtitvinf.h +3 -3
  266. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/dtptngen.h +2 -2
  267. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/dtrule.h +2 -2
  268. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/fieldpos.h +4 -4
  269. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/fmtable.h +2 -2
  270. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/format.h +2 -2
  271. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/fpositer.h +2 -2
  272. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/measfmt.h +1 -1
  273. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/measunit.h +2 -2
  274. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/measure.h +1 -1
  275. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/msgfmt.h +2 -2
  276. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/numfmt.h +1 -1
  277. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/plurfmt.h +2 -2
  278. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/plurrule.h +2 -2
  279. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/rbnf.h +1 -1
  280. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/rbtz.h +2 -2
  281. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/region.h +2 -2
  282. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/search.h +2 -2
  283. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/selfmt.h +2 -2
  284. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/simpletz.h +1 -1
  285. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/smpdtfmt.h +1 -1
  286. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/sortkey.h +3 -3
  287. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/stsearch.h +1 -1
  288. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/tblcoll.h +1 -1
  289. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/timezone.h +2 -2
  290. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/tmutamt.h +3 -3
  291. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/tmutfmt.h +2 -2
  292. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/tzfmt.h +1 -1
  293. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/tznames.h +2 -2
  294. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/tzrule.h +8 -8
  295. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/tztrans.h +2 -2
  296. package/src/duckdb/extension/icu/third_party/icu/i18n/unicode/vtzone.h +2 -2
  297. package/src/duckdb/extension/icu/third_party/icu/i18n/utf16collationiterator.cpp +2 -2
  298. package/src/duckdb/extension/icu/third_party/icu/i18n/utf16collationiterator.h +2 -2
  299. package/src/duckdb/extension/icu/third_party/icu/i18n/vtzone.cpp +2 -2
  300. package/src/duckdb/extension/json/buffered_json_reader.cpp +6 -1
  301. package/src/duckdb/extension/json/include/buffered_json_reader.hpp +2 -0
  302. package/src/duckdb/extension/json/include/json_common.hpp +14 -10
  303. package/src/duckdb/extension/json/include/json_scan.hpp +48 -7
  304. package/src/duckdb/extension/json/include/json_structure.hpp +2 -1
  305. package/src/duckdb/extension/json/include/json_transform.hpp +5 -2
  306. package/src/duckdb/extension/json/json_functions/copy_json.cpp +1 -1
  307. package/src/duckdb/extension/json/json_functions/json_create.cpp +57 -20
  308. package/src/duckdb/extension/json/json_functions/json_serialize_plan.cpp +7 -6
  309. package/src/duckdb/extension/json/json_functions/json_serialize_sql.cpp +6 -5
  310. package/src/duckdb/extension/json/json_functions/json_structure.cpp +20 -17
  311. package/src/duckdb/extension/json/json_functions/json_transform.cpp +48 -17
  312. package/src/duckdb/extension/json/json_functions/read_json.cpp +83 -34
  313. package/src/duckdb/extension/json/json_functions/read_json_objects.cpp +3 -3
  314. package/src/duckdb/extension/json/json_functions.cpp +14 -16
  315. package/src/duckdb/extension/json/json_scan.cpp +36 -16
  316. package/src/duckdb/extension/json/json_serializer.cpp +1 -1
  317. package/src/duckdb/extension/json/serialize_json.cpp +2 -2
  318. package/src/duckdb/extension/parquet/column_reader.cpp +136 -116
  319. package/src/duckdb/extension/parquet/column_writer.cpp +870 -604
  320. package/src/duckdb/extension/parquet/geo_parquet.cpp +4 -5
  321. package/src/duckdb/extension/parquet/include/boolean_column_reader.hpp +0 -4
  322. package/src/duckdb/extension/parquet/include/column_reader.hpp +24 -19
  323. package/src/duckdb/extension/parquet/include/column_writer.hpp +7 -5
  324. package/src/duckdb/extension/parquet/include/decode_utils.hpp +138 -18
  325. package/src/duckdb/extension/parquet/include/geo_parquet.hpp +4 -3
  326. package/src/duckdb/extension/parquet/include/null_column_reader.hpp +1 -14
  327. package/src/duckdb/extension/parquet/include/parquet_bss_encoder.hpp +45 -0
  328. package/src/duckdb/extension/parquet/include/parquet_crypto.hpp +1 -1
  329. package/src/duckdb/extension/parquet/include/parquet_dbp_decoder.hpp +101 -90
  330. package/src/duckdb/extension/parquet/include/parquet_dbp_encoder.hpp +179 -0
  331. package/src/duckdb/extension/parquet/include/parquet_decimal_utils.hpp +2 -3
  332. package/src/duckdb/extension/parquet/include/parquet_dlba_encoder.hpp +48 -0
  333. package/src/duckdb/extension/parquet/include/parquet_extension.hpp +8 -0
  334. package/src/duckdb/extension/parquet/include/parquet_file_metadata_cache.hpp +2 -2
  335. package/src/duckdb/extension/parquet/include/parquet_metadata.hpp +5 -0
  336. package/src/duckdb/extension/parquet/include/parquet_reader.hpp +22 -18
  337. package/src/duckdb/extension/parquet/include/parquet_rle_bp_decoder.hpp +1 -5
  338. package/src/duckdb/extension/parquet/include/parquet_statistics.hpp +87 -3
  339. package/src/duckdb/extension/parquet/include/parquet_writer.hpp +30 -16
  340. package/src/duckdb/extension/parquet/include/resizable_buffer.hpp +1 -0
  341. package/src/duckdb/extension/parquet/include/string_column_reader.hpp +0 -8
  342. package/src/duckdb/extension/parquet/include/struct_column_reader.hpp +1 -1
  343. package/src/duckdb/extension/parquet/include/templated_column_reader.hpp +1 -42
  344. package/src/duckdb/extension/parquet/include/thrift_tools.hpp +13 -1
  345. package/src/duckdb/extension/parquet/include/zstd_file_system.hpp +4 -0
  346. package/src/duckdb/extension/parquet/parquet_extension.cpp +240 -197
  347. package/src/duckdb/extension/parquet/parquet_metadata.cpp +138 -6
  348. package/src/duckdb/extension/parquet/parquet_reader.cpp +155 -79
  349. package/src/duckdb/extension/parquet/parquet_statistics.cpp +258 -38
  350. package/src/duckdb/extension/parquet/parquet_timestamp.cpp +17 -3
  351. package/src/duckdb/extension/parquet/parquet_writer.cpp +65 -34
  352. package/src/duckdb/extension/parquet/serialize_parquet.cpp +4 -0
  353. package/src/duckdb/extension/parquet/zstd_file_system.cpp +13 -0
  354. package/src/duckdb/src/catalog/catalog.cpp +272 -97
  355. package/src/duckdb/src/catalog/catalog_entry/duck_index_entry.cpp +9 -4
  356. package/src/duckdb/src/catalog/catalog_entry/duck_schema_entry.cpp +8 -0
  357. package/src/duckdb/src/catalog/catalog_entry/duck_table_entry.cpp +145 -95
  358. package/src/duckdb/src/catalog/catalog_entry/scalar_function_catalog_entry.cpp +9 -3
  359. package/src/duckdb/src/catalog/catalog_entry/schema_catalog_entry.cpp +15 -0
  360. package/src/duckdb/src/catalog/catalog_entry/table_catalog_entry.cpp +40 -24
  361. package/src/duckdb/src/catalog/catalog_entry/table_function_catalog_entry.cpp +1 -1
  362. package/src/duckdb/src/catalog/catalog_entry/type_catalog_entry.cpp +2 -2
  363. package/src/duckdb/src/catalog/catalog_entry.cpp +3 -0
  364. package/src/duckdb/src/catalog/catalog_entry_retriever.cpp +60 -5
  365. package/src/duckdb/src/catalog/catalog_search_path.cpp +27 -14
  366. package/src/duckdb/src/catalog/catalog_set.cpp +75 -31
  367. package/src/duckdb/src/catalog/default/default_functions.cpp +13 -8
  368. package/src/duckdb/src/catalog/default/default_views.cpp +1 -0
  369. package/src/duckdb/src/catalog/dependency_manager.cpp +133 -5
  370. package/src/duckdb/src/catalog/duck_catalog.cpp +17 -9
  371. package/src/duckdb/src/common/adbc/adbc.cpp +18 -0
  372. package/src/duckdb/src/common/allocator.cpp +3 -1
  373. package/src/duckdb/src/common/arrow/arrow_appender.cpp +30 -9
  374. package/src/duckdb/src/common/arrow/arrow_converter.cpp +63 -82
  375. package/src/duckdb/src/common/arrow/arrow_merge_event.cpp +4 -3
  376. package/src/duckdb/src/common/arrow/arrow_type_extension.cpp +361 -0
  377. package/src/duckdb/src/common/arrow/arrow_util.cpp +10 -6
  378. package/src/duckdb/src/common/arrow/arrow_wrapper.cpp +6 -2
  379. package/src/duckdb/src/common/arrow/physical_arrow_collector.cpp +2 -1
  380. package/src/duckdb/src/common/arrow/schema_metadata.cpp +27 -14
  381. package/src/duckdb/src/common/assert.cpp +1 -2
  382. package/src/duckdb/src/common/bind_helpers.cpp +1 -1
  383. package/src/duckdb/src/common/box_renderer.cpp +316 -26
  384. package/src/duckdb/src/common/cgroups.cpp +7 -1
  385. package/src/duckdb/src/common/compressed_file_system.cpp +1 -1
  386. package/src/duckdb/src/common/enum_util.cpp +2865 -6882
  387. package/src/duckdb/src/common/enums/compression_type.cpp +12 -0
  388. package/src/duckdb/src/common/enums/metric_type.cpp +24 -0
  389. package/src/duckdb/src/common/enums/optimizer_type.cpp +4 -0
  390. package/src/duckdb/src/common/enums/physical_operator_type.cpp +2 -0
  391. package/src/duckdb/src/common/error_data.cpp +23 -6
  392. package/src/duckdb/src/common/exception/binder_exception.cpp +1 -1
  393. package/src/duckdb/src/common/exception.cpp +20 -28
  394. package/src/duckdb/src/common/extra_type_info.cpp +85 -20
  395. package/src/duckdb/src/common/file_buffer.cpp +5 -2
  396. package/src/duckdb/src/common/file_system.cpp +8 -3
  397. package/src/duckdb/src/common/fsst.cpp +3 -3
  398. package/src/duckdb/src/common/hive_partitioning.cpp +1 -1
  399. package/src/duckdb/src/common/local_file_system.cpp +169 -60
  400. package/src/duckdb/src/common/multi_file_list.cpp +4 -1
  401. package/src/duckdb/src/common/multi_file_reader.cpp +240 -63
  402. package/src/duckdb/src/common/opener_file_system.cpp +37 -0
  403. package/src/duckdb/src/common/operator/cast_operators.cpp +77 -11
  404. package/src/duckdb/src/common/operator/string_cast.cpp +6 -2
  405. package/src/duckdb/src/common/pipe_file_system.cpp +4 -4
  406. package/src/duckdb/src/common/progress_bar/progress_bar.cpp +25 -14
  407. package/src/duckdb/src/common/radix_partitioning.cpp +17 -16
  408. package/src/duckdb/src/common/random_engine.cpp +39 -3
  409. package/src/duckdb/src/common/render_tree.cpp +3 -19
  410. package/src/duckdb/src/common/row_operations/row_external.cpp +1 -1
  411. package/src/duckdb/src/common/row_operations/row_gather.cpp +2 -58
  412. package/src/duckdb/src/common/row_operations/row_matcher.cpp +2 -2
  413. package/src/duckdb/src/common/row_operations/row_radix_scatter.cpp +2 -0
  414. package/src/duckdb/src/common/row_operations/row_scatter.cpp +20 -19
  415. package/src/duckdb/src/common/serializer/buffered_file_writer.cpp +1 -1
  416. package/src/duckdb/src/common/serializer/memory_stream.cpp +36 -0
  417. package/src/duckdb/src/common/sort/comparators.cpp +7 -7
  418. package/src/duckdb/src/common/sort/partition_state.cpp +2 -2
  419. package/src/duckdb/src/common/stacktrace.cpp +127 -0
  420. package/src/duckdb/src/common/string_util.cpp +157 -32
  421. package/src/duckdb/src/common/tree_renderer/text_tree_renderer.cpp +15 -3
  422. package/src/duckdb/src/common/types/column/column_data_allocator.cpp +4 -0
  423. package/src/duckdb/src/common/types/column/column_data_collection.cpp +71 -8
  424. package/src/duckdb/src/common/types/column/column_data_collection_segment.cpp +27 -6
  425. package/src/duckdb/src/common/types/conflict_manager.cpp +21 -7
  426. package/src/duckdb/src/common/types/date.cpp +39 -25
  427. package/src/duckdb/src/common/types/row/partitioned_tuple_data.cpp +4 -11
  428. package/src/duckdb/src/common/types/row/tuple_data_allocator.cpp +21 -7
  429. package/src/duckdb/src/common/types/row/tuple_data_collection.cpp +10 -1
  430. package/src/duckdb/src/common/types/row/tuple_data_scatter_gather.cpp +17 -17
  431. package/src/duckdb/src/common/types/timestamp.cpp +70 -33
  432. package/src/duckdb/src/common/types/uuid.cpp +11 -0
  433. package/src/duckdb/src/common/types/validity_mask.cpp +16 -5
  434. package/src/duckdb/src/common/types/value.cpp +357 -199
  435. package/src/duckdb/src/common/types/varint.cpp +64 -18
  436. package/src/duckdb/src/common/types/vector.cpp +78 -38
  437. package/src/duckdb/src/common/types.cpp +199 -92
  438. package/src/duckdb/src/common/vector_operations/comparison_operators.cpp +2 -1
  439. package/src/duckdb/src/common/vector_operations/is_distinct_from.cpp +32 -5
  440. package/src/duckdb/src/common/vector_operations/vector_hash.cpp +3 -1
  441. package/src/duckdb/src/execution/adaptive_filter.cpp +6 -2
  442. package/src/duckdb/src/execution/aggregate_hashtable.cpp +410 -111
  443. package/src/duckdb/src/execution/column_binding_resolver.cpp +2 -2
  444. package/src/duckdb/src/execution/expression_executor/execute_between.cpp +6 -0
  445. package/src/duckdb/src/execution/expression_executor/execute_cast.cpp +4 -3
  446. package/src/duckdb/src/execution/expression_executor/execute_comparison.cpp +2 -2
  447. package/src/duckdb/src/execution/expression_executor/execute_conjunction.cpp +2 -2
  448. package/src/duckdb/src/execution/expression_executor/execute_function.cpp +1 -0
  449. package/src/duckdb/src/execution/expression_executor/execute_operator.cpp +5 -4
  450. package/src/duckdb/src/execution/expression_executor.cpp +5 -3
  451. package/src/duckdb/src/execution/index/art/art.cpp +208 -72
  452. package/src/duckdb/src/execution/index/art/base_leaf.cpp +1 -1
  453. package/src/duckdb/src/execution/index/art/leaf.cpp +12 -7
  454. package/src/duckdb/src/execution/index/art/node.cpp +2 -1
  455. package/src/duckdb/src/execution/index/art/node256_leaf.cpp +6 -6
  456. package/src/duckdb/src/execution/index/art/plan_art.cpp +50 -55
  457. package/src/duckdb/src/execution/index/art/prefix.cpp +7 -13
  458. package/src/duckdb/src/execution/index/bound_index.cpp +30 -5
  459. package/src/duckdb/src/execution/index/fixed_size_allocator.cpp +3 -5
  460. package/src/duckdb/src/execution/index/fixed_size_buffer.cpp +14 -9
  461. package/src/duckdb/src/execution/join_hashtable.cpp +254 -158
  462. package/src/duckdb/src/execution/operator/aggregate/grouped_aggregate_data.cpp +1 -1
  463. package/src/duckdb/src/execution/operator/aggregate/physical_hash_aggregate.cpp +7 -7
  464. package/src/duckdb/src/execution/operator/aggregate/physical_partitioned_aggregate.cpp +226 -0
  465. package/src/duckdb/src/execution/operator/aggregate/physical_perfecthash_aggregate.cpp +3 -3
  466. package/src/duckdb/src/execution/operator/aggregate/physical_streaming_window.cpp +3 -3
  467. package/src/duckdb/src/execution/operator/aggregate/physical_ungrouped_aggregate.cpp +77 -70
  468. package/src/duckdb/src/execution/operator/aggregate/physical_window.cpp +114 -50
  469. package/src/duckdb/src/execution/operator/csv_scanner/buffer_manager/csv_buffer.cpp +2 -2
  470. package/src/duckdb/src/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.cpp +19 -10
  471. package/src/duckdb/src/execution/operator/csv_scanner/buffer_manager/csv_file_handle.cpp +22 -15
  472. package/src/duckdb/src/execution/operator/csv_scanner/encode/csv_encoder.cpp +95 -0
  473. package/src/duckdb/src/execution/operator/csv_scanner/scanner/column_count_scanner.cpp +6 -1
  474. package/src/duckdb/src/execution/operator/csv_scanner/scanner/csv_schema.cpp +75 -2
  475. package/src/duckdb/src/execution/operator/csv_scanner/scanner/scanner_boundary.cpp +40 -12
  476. package/src/duckdb/src/execution/operator/csv_scanner/scanner/string_value_scanner.cpp +395 -163
  477. package/src/duckdb/src/execution/operator/csv_scanner/sniffer/csv_sniffer.cpp +20 -23
  478. package/src/duckdb/src/execution/operator/csv_scanner/sniffer/dialect_detection.cpp +115 -49
  479. package/src/duckdb/src/execution/operator/csv_scanner/sniffer/header_detection.cpp +66 -12
  480. package/src/duckdb/src/execution/operator/csv_scanner/sniffer/type_detection.cpp +20 -23
  481. package/src/duckdb/src/execution/operator/csv_scanner/state_machine/csv_state_machine_cache.cpp +220 -46
  482. package/src/duckdb/src/execution/operator/csv_scanner/table_function/csv_file_scanner.cpp +43 -32
  483. package/src/duckdb/src/execution/operator/csv_scanner/table_function/global_csv_state.cpp +54 -119
  484. package/src/duckdb/src/execution/operator/csv_scanner/util/csv_error.cpp +184 -20
  485. package/src/duckdb/src/execution/operator/csv_scanner/util/csv_reader_options.cpp +83 -21
  486. package/src/duckdb/src/execution/operator/csv_scanner/util/csv_validator.cpp +63 -0
  487. package/src/duckdb/src/execution/operator/helper/physical_reservoir_sample.cpp +7 -4
  488. package/src/duckdb/src/execution/operator/helper/physical_set.cpp +1 -1
  489. package/src/duckdb/src/execution/operator/helper/physical_vacuum.cpp +3 -2
  490. package/src/duckdb/src/execution/operator/helper/physical_verify_vector.cpp +9 -1
  491. package/src/duckdb/src/execution/operator/join/perfect_hash_join_executor.cpp +132 -15
  492. package/src/duckdb/src/execution/operator/join/physical_blockwise_nl_join.cpp +64 -55
  493. package/src/duckdb/src/execution/operator/join/physical_hash_join.cpp +284 -154
  494. package/src/duckdb/src/execution/operator/join/physical_iejoin.cpp +40 -55
  495. package/src/duckdb/src/execution/operator/join/physical_range_join.cpp +0 -1
  496. package/src/duckdb/src/execution/operator/order/physical_order.cpp +7 -3
  497. package/src/duckdb/src/execution/operator/order/physical_top_n.cpp +298 -227
  498. package/src/duckdb/src/execution/operator/persistent/csv_rejects_table.cpp +5 -2
  499. package/src/duckdb/src/execution/operator/persistent/physical_batch_copy_to_file.cpp +3 -4
  500. package/src/duckdb/src/execution/operator/persistent/physical_batch_insert.cpp +32 -19
  501. package/src/duckdb/src/execution/operator/persistent/physical_copy_database.cpp +1 -0
  502. package/src/duckdb/src/execution/operator/persistent/physical_copy_to_file.cpp +6 -0
  503. package/src/duckdb/src/execution/operator/persistent/physical_delete.cpp +58 -19
  504. package/src/duckdb/src/execution/operator/persistent/physical_export.cpp +11 -27
  505. package/src/duckdb/src/execution/operator/persistent/physical_insert.cpp +308 -119
  506. package/src/duckdb/src/execution/operator/persistent/physical_update.cpp +105 -55
  507. package/src/duckdb/src/execution/operator/projection/physical_tableinout_function.cpp +6 -2
  508. package/src/duckdb/src/execution/operator/projection/physical_unnest.cpp +1 -1
  509. package/src/duckdb/src/execution/operator/scan/physical_positional_scan.cpp +15 -6
  510. package/src/duckdb/src/execution/operator/scan/physical_table_scan.cpp +92 -50
  511. package/src/duckdb/src/execution/operator/schema/physical_alter.cpp +0 -1
  512. package/src/duckdb/src/execution/operator/schema/physical_attach.cpp +8 -4
  513. package/src/duckdb/src/execution/operator/schema/physical_create_art_index.cpp +54 -22
  514. package/src/duckdb/src/execution/operator/set/physical_union.cpp +5 -1
  515. package/src/duckdb/src/execution/physical_operator.cpp +15 -9
  516. package/src/duckdb/src/execution/physical_plan/plan_aggregate.cpp +101 -12
  517. package/src/duckdb/src/execution/physical_plan/plan_comparison_join.cpp +11 -140
  518. package/src/duckdb/src/execution/physical_plan/plan_create_index.cpp +11 -13
  519. package/src/duckdb/src/execution/physical_plan/plan_cte.cpp +1 -1
  520. package/src/duckdb/src/execution/physical_plan/plan_delete.cpp +1 -1
  521. package/src/duckdb/src/execution/physical_plan/plan_delim_join.cpp +1 -1
  522. package/src/duckdb/src/execution/physical_plan/plan_distinct.cpp +6 -5
  523. package/src/duckdb/src/execution/physical_plan/plan_export.cpp +0 -4
  524. package/src/duckdb/src/execution/physical_plan/plan_filter.cpp +1 -1
  525. package/src/duckdb/src/execution/physical_plan/plan_get.cpp +16 -13
  526. package/src/duckdb/src/execution/physical_plan/plan_insert.cpp +1 -1
  527. package/src/duckdb/src/execution/physical_plan/plan_order.cpp +7 -7
  528. package/src/duckdb/src/execution/physical_plan/plan_prepare.cpp +2 -2
  529. package/src/duckdb/src/execution/physical_plan/plan_projection.cpp +1 -1
  530. package/src/duckdb/src/execution/physical_plan/plan_sample.cpp +8 -3
  531. package/src/duckdb/src/execution/physical_plan/plan_set_operation.cpp +1 -2
  532. package/src/duckdb/src/execution/physical_plan/plan_simple.cpp +1 -2
  533. package/src/duckdb/src/execution/physical_plan/plan_top_n.cpp +3 -2
  534. package/src/duckdb/src/execution/physical_plan_generator.cpp +0 -22
  535. package/src/duckdb/src/execution/radix_partitioned_hashtable.cpp +136 -116
  536. package/src/duckdb/src/execution/sample/base_reservoir_sample.cpp +136 -0
  537. package/src/duckdb/src/execution/sample/reservoir_sample.cpp +930 -0
  538. package/src/duckdb/src/function/aggregate/distributive/count.cpp +6 -12
  539. package/src/duckdb/src/function/aggregate/distributive/{first.cpp → first_last_any.cpp} +37 -18
  540. package/src/duckdb/src/{core_functions → function}/aggregate/distributive/minmax.cpp +19 -12
  541. package/src/duckdb/src/function/aggregate/sorted_aggregate_function.cpp +72 -13
  542. package/src/duckdb/src/function/built_in_functions.cpp +85 -2
  543. package/src/duckdb/src/function/cast/decimal_cast.cpp +1 -1
  544. package/src/duckdb/src/function/cast/string_cast.cpp +1 -1
  545. package/src/duckdb/src/function/cast/struct_cast.cpp +81 -49
  546. package/src/duckdb/src/function/cast/union/from_struct.cpp +7 -5
  547. package/src/duckdb/src/function/compression_config.cpp +6 -0
  548. package/src/duckdb/src/function/encoding_function.cpp +134 -0
  549. package/src/duckdb/src/function/function.cpp +8 -13
  550. package/src/duckdb/src/function/function_binder.cpp +100 -21
  551. package/src/duckdb/src/function/function_list.cpp +178 -0
  552. package/src/duckdb/src/function/macro_function.cpp +4 -4
  553. package/src/duckdb/src/function/pragma/pragma_functions.cpp +0 -2
  554. package/src/duckdb/src/function/pragma/pragma_queries.cpp +0 -4
  555. package/src/duckdb/src/{core_functions/core_functions.cpp → function/register_function_list.cpp} +12 -8
  556. package/src/duckdb/src/function/scalar/compressed_materialization/compress_integral.cpp +62 -23
  557. package/src/duckdb/src/function/scalar/compressed_materialization/compress_string.cpp +33 -16
  558. package/src/duckdb/src/function/scalar/compressed_materialization_utils.cpp +21 -0
  559. package/src/duckdb/src/{core_functions/scalar/blob → function/scalar}/create_sort_key.cpp +86 -23
  560. package/src/duckdb/src/{core_functions → function}/scalar/date/strftime.cpp +6 -4
  561. package/src/duckdb/src/function/scalar/generic/constant_or_null.cpp +5 -7
  562. package/src/duckdb/src/{core_functions → function}/scalar/generic/error.cpp +3 -1
  563. package/src/duckdb/src/function/scalar/generic/getvariable.cpp +2 -2
  564. package/src/duckdb/src/function/scalar/list/contains_or_position.cpp +1 -7
  565. package/src/duckdb/src/function/scalar/list/list_extract.cpp +27 -21
  566. package/src/duckdb/src/function/scalar/list/list_resize.cpp +8 -12
  567. package/src/duckdb/src/function/scalar/list/list_select.cpp +1 -4
  568. package/src/duckdb/src/function/scalar/list/list_zip.cpp +6 -6
  569. package/src/duckdb/src/{core_functions → function}/scalar/map/map_contains.cpp +2 -2
  570. package/src/duckdb/src/function/scalar/nested_functions.cpp +0 -11
  571. package/src/duckdb/src/function/scalar/{operators → operator}/add.cpp +2 -1
  572. package/src/duckdb/src/function/scalar/{operators → operator}/arithmetic.cpp +195 -127
  573. package/src/duckdb/src/function/scalar/sequence/nextval.cpp +30 -21
  574. package/src/duckdb/src/function/scalar/strftime_format.cpp +10 -0
  575. package/src/duckdb/src/function/scalar/string/caseconvert.cpp +11 -41
  576. package/src/duckdb/src/function/scalar/string/concat.cpp +22 -20
  577. package/src/duckdb/src/function/scalar/string/concat_ws.cpp +2 -2
  578. package/src/duckdb/src/function/scalar/string/contains.cpp +16 -19
  579. package/src/duckdb/src/function/scalar/string/length.cpp +38 -24
  580. package/src/duckdb/src/function/scalar/string/like.cpp +80 -47
  581. package/src/duckdb/src/{core_functions → function}/scalar/string/md5.cpp +2 -2
  582. package/src/duckdb/src/function/scalar/string/nfc_normalize.cpp +2 -6
  583. package/src/duckdb/src/function/scalar/string/prefix.cpp +0 -4
  584. package/src/duckdb/src/function/scalar/string/regexp/regexp_extract_all.cpp +2 -1
  585. package/src/duckdb/src/function/scalar/string/regexp.cpp +17 -7
  586. package/src/duckdb/src/{core_functions → function}/scalar/string/regexp_escape.cpp +2 -2
  587. package/src/duckdb/src/{core_functions → function}/scalar/string/sha1.cpp +1 -1
  588. package/src/duckdb/src/{core_functions → function}/scalar/string/sha256.cpp +1 -1
  589. package/src/duckdb/src/{core_functions → function}/scalar/string/string_split.cpp +4 -5
  590. package/src/duckdb/src/function/scalar/string/strip_accents.cpp +3 -6
  591. package/src/duckdb/src/function/scalar/string/substring.cpp +14 -13
  592. package/src/duckdb/src/function/scalar/string/suffix.cpp +0 -4
  593. package/src/duckdb/src/function/scalar/struct/struct_concat.cpp +115 -0
  594. package/src/duckdb/src/function/scalar/struct/struct_extract.cpp +35 -31
  595. package/src/duckdb/src/{core_functions → function}/scalar/struct/struct_pack.cpp +7 -7
  596. package/src/duckdb/src/function/scalar/system/aggregate_export.cpp +5 -8
  597. package/src/duckdb/src/function/scalar/system/write_log.cpp +170 -0
  598. package/src/duckdb/src/function/scalar_function.cpp +5 -5
  599. package/src/duckdb/src/function/table/arrow/arrow_array_scan_state.cpp +3 -2
  600. package/src/duckdb/src/function/table/arrow/arrow_duck_schema.cpp +287 -1
  601. package/src/duckdb/src/function/table/arrow/arrow_type_info.cpp +6 -6
  602. package/src/duckdb/src/function/table/arrow.cpp +32 -352
  603. package/src/duckdb/src/function/table/arrow_conversion.cpp +43 -7
  604. package/src/duckdb/src/function/table/copy_csv.cpp +38 -23
  605. package/src/duckdb/src/function/table/glob.cpp +1 -1
  606. package/src/duckdb/src/function/table/query_function.cpp +12 -7
  607. package/src/duckdb/src/function/table/read_csv.cpp +114 -46
  608. package/src/duckdb/src/function/table/read_file.cpp +26 -6
  609. package/src/duckdb/src/function/table/sniff_csv.cpp +25 -5
  610. package/src/duckdb/src/function/table/system/duckdb_columns.cpp +1 -1
  611. package/src/duckdb/src/function/table/system/duckdb_constraints.cpp +1 -1
  612. package/src/duckdb/src/function/table/system/duckdb_dependencies.cpp +6 -7
  613. package/src/duckdb/src/function/table/system/duckdb_extensions.cpp +1 -1
  614. package/src/duckdb/src/function/table/system/duckdb_functions.cpp +141 -16
  615. package/src/duckdb/src/function/table/system/duckdb_log.cpp +64 -0
  616. package/src/duckdb/src/function/table/system/duckdb_log_contexts.cpp +65 -0
  617. package/src/duckdb/src/function/table/system/duckdb_memory.cpp +0 -1
  618. package/src/duckdb/src/function/table/system/duckdb_settings.cpp +1 -1
  619. package/src/duckdb/src/function/table/system/duckdb_tables.cpp +1 -13
  620. package/src/duckdb/src/function/table/system/duckdb_types.cpp +1 -1
  621. package/src/duckdb/src/function/table/system/pragma_storage_info.cpp +17 -0
  622. package/src/duckdb/src/function/table/system/pragma_table_info.cpp +6 -0
  623. package/src/duckdb/src/function/table/system/pragma_table_sample.cpp +95 -0
  624. package/src/duckdb/src/function/table/system/test_all_types.cpp +56 -46
  625. package/src/duckdb/src/function/table/system_functions.cpp +3 -0
  626. package/src/duckdb/src/function/table/table_scan.cpp +487 -289
  627. package/src/duckdb/src/function/table/version/pragma_version.cpp +3 -3
  628. package/src/duckdb/src/function/table_function.cpp +10 -6
  629. package/src/duckdb/src/function/window/window_aggregate_function.cpp +248 -0
  630. package/src/duckdb/src/function/window/window_aggregate_states.cpp +48 -0
  631. package/src/duckdb/src/function/window/window_aggregator.cpp +88 -0
  632. package/src/duckdb/src/function/window/window_boundaries_state.cpp +854 -0
  633. package/src/duckdb/src/function/window/window_collection.cpp +146 -0
  634. package/src/duckdb/src/function/window/window_constant_aggregator.cpp +357 -0
  635. package/src/duckdb/src/function/window/window_custom_aggregator.cpp +146 -0
  636. package/src/duckdb/src/function/window/window_distinct_aggregator.cpp +758 -0
  637. package/src/duckdb/src/function/window/window_executor.cpp +99 -0
  638. package/src/duckdb/src/function/window/window_index_tree.cpp +63 -0
  639. package/src/duckdb/src/function/window/window_merge_sort_tree.cpp +275 -0
  640. package/src/duckdb/src/function/window/window_naive_aggregator.cpp +361 -0
  641. package/src/duckdb/src/function/window/window_rank_function.cpp +288 -0
  642. package/src/duckdb/src/function/window/window_rownumber_function.cpp +191 -0
  643. package/src/duckdb/src/function/window/window_segment_tree.cpp +594 -0
  644. package/src/duckdb/src/function/window/window_shared_expressions.cpp +50 -0
  645. package/src/duckdb/src/function/window/window_token_tree.cpp +142 -0
  646. package/src/duckdb/src/function/window/window_value_function.cpp +566 -0
  647. package/src/duckdb/src/include/duckdb/catalog/catalog.hpp +74 -17
  648. package/src/duckdb/src/include/duckdb/catalog/catalog_entry/duck_index_entry.hpp +1 -1
  649. package/src/duckdb/src/include/duckdb/catalog/catalog_entry/duck_schema_entry.hpp +2 -0
  650. package/src/duckdb/src/include/duckdb/catalog/catalog_entry/duck_table_entry.hpp +9 -0
  651. package/src/duckdb/src/include/duckdb/catalog/catalog_entry/function_entry.hpp +4 -10
  652. package/src/duckdb/src/include/duckdb/catalog/catalog_entry/pragma_function_catalog_entry.hpp +1 -1
  653. package/src/duckdb/src/include/duckdb/catalog/catalog_entry/scalar_function_catalog_entry.hpp +2 -2
  654. package/src/duckdb/src/include/duckdb/catalog/catalog_entry/schema_catalog_entry.hpp +2 -0
  655. package/src/duckdb/src/include/duckdb/catalog/catalog_entry/table_catalog_entry.hpp +18 -3
  656. package/src/duckdb/src/include/duckdb/catalog/catalog_entry/table_function_catalog_entry.hpp +1 -1
  657. package/src/duckdb/src/include/duckdb/catalog/catalog_entry/type_catalog_entry.hpp +2 -1
  658. package/src/duckdb/src/include/duckdb/catalog/catalog_entry.hpp +5 -2
  659. package/src/duckdb/src/include/duckdb/catalog/catalog_entry_retriever.hpp +21 -18
  660. package/src/duckdb/src/include/duckdb/catalog/catalog_search_path.hpp +3 -2
  661. package/src/duckdb/src/include/duckdb/catalog/catalog_set.hpp +10 -2
  662. package/src/duckdb/src/include/duckdb/catalog/dependency_manager.hpp +11 -0
  663. package/src/duckdb/src/include/duckdb/catalog/duck_catalog.hpp +9 -4
  664. package/src/duckdb/src/include/duckdb/common/allocator.hpp +3 -0
  665. package/src/duckdb/src/include/duckdb/common/array_ptr.hpp +8 -0
  666. package/src/duckdb/src/include/duckdb/common/arrow/appender/append_data.hpp +4 -1
  667. package/src/duckdb/src/include/duckdb/common/arrow/appender/list_data.hpp +3 -1
  668. package/src/duckdb/src/include/duckdb/common/arrow/appender/list_view_data.hpp +3 -1
  669. package/src/duckdb/src/include/duckdb/common/arrow/appender/varchar_data.hpp +2 -1
  670. package/src/duckdb/src/include/duckdb/common/arrow/arrow_appender.hpp +7 -3
  671. package/src/duckdb/src/include/duckdb/common/arrow/arrow_converter.hpp +26 -3
  672. package/src/duckdb/src/include/duckdb/common/arrow/arrow_query_result.hpp +1 -1
  673. package/src/duckdb/src/include/duckdb/common/arrow/arrow_type_extension.hpp +144 -0
  674. package/src/duckdb/src/include/duckdb/common/arrow/arrow_util.hpp +5 -2
  675. package/src/duckdb/src/include/duckdb/common/arrow/result_arrow_wrapper.hpp +2 -0
  676. package/src/duckdb/src/include/duckdb/common/arrow/schema_metadata.hpp +11 -4
  677. package/src/duckdb/src/include/duckdb/common/assert.hpp +12 -1
  678. package/src/duckdb/src/include/duckdb/common/atomic_ptr.hpp +102 -0
  679. package/src/duckdb/src/include/duckdb/common/box_renderer.hpp +65 -6
  680. package/src/duckdb/src/include/duckdb/common/chrono.hpp +1 -0
  681. package/src/duckdb/src/include/duckdb/common/column_index.hpp +72 -0
  682. package/src/duckdb/src/include/duckdb/common/enum_util.hpp +120 -0
  683. package/src/duckdb/src/include/duckdb/{core_functions/core_functions.hpp → common/enums/collation_type.hpp} +2 -7
  684. package/src/duckdb/src/include/duckdb/common/enums/compression_type.hpp +5 -2
  685. package/src/duckdb/src/include/duckdb/common/enums/function_errors.hpp +18 -0
  686. package/src/duckdb/src/include/duckdb/common/enums/memory_tag.hpp +3 -2
  687. package/src/duckdb/src/include/duckdb/common/enums/metric_type.hpp +7 -2
  688. package/src/duckdb/src/include/duckdb/common/enums/optimizer_type.hpp +4 -0
  689. package/src/duckdb/src/include/duckdb/common/enums/order_preservation_type.hpp +1 -1
  690. package/src/duckdb/src/include/duckdb/common/enums/physical_operator_type.hpp +1 -0
  691. package/src/duckdb/src/include/duckdb/common/enums/profiler_format.hpp +1 -1
  692. package/src/duckdb/src/include/duckdb/{core_functions/aggregate → common/enums}/quantile_enum.hpp +3 -1
  693. package/src/duckdb/src/include/duckdb/common/enums/scan_vector_type.hpp +2 -0
  694. package/src/duckdb/src/include/duckdb/common/error_data.hpp +1 -0
  695. package/src/duckdb/src/include/duckdb/common/exception/parser_exception.hpp +4 -0
  696. package/src/duckdb/src/include/duckdb/common/exception.hpp +1 -1
  697. package/src/duckdb/src/include/duckdb/common/extension_type_info.hpp +37 -0
  698. package/src/duckdb/src/include/duckdb/common/extra_operator_info.hpp +7 -2
  699. package/src/duckdb/src/include/duckdb/common/extra_type_info.hpp +9 -3
  700. package/src/duckdb/src/include/duckdb/common/fast_mem.hpp +6 -6
  701. package/src/duckdb/src/include/duckdb/common/file_buffer.hpp +19 -10
  702. package/src/duckdb/src/include/duckdb/common/file_opener.hpp +2 -0
  703. package/src/duckdb/src/include/duckdb/common/file_system.hpp +6 -1
  704. package/src/duckdb/src/include/duckdb/common/fsst.hpp +2 -2
  705. package/src/duckdb/src/include/duckdb/common/helper.hpp +6 -0
  706. package/src/duckdb/src/include/duckdb/common/hugeint.hpp +10 -0
  707. package/src/duckdb/src/include/duckdb/common/insertion_order_preserving_map.hpp +12 -2
  708. package/src/duckdb/src/include/duckdb/common/local_file_system.hpp +3 -0
  709. package/src/duckdb/src/include/duckdb/common/multi_file_list.hpp +2 -1
  710. package/src/duckdb/src/include/duckdb/common/multi_file_reader.hpp +147 -27
  711. package/src/duckdb/src/include/duckdb/common/multi_file_reader_options.hpp +4 -0
  712. package/src/duckdb/src/include/duckdb/common/numeric_utils.hpp +2 -7
  713. package/src/duckdb/src/include/duckdb/common/opener_file_system.hpp +16 -5
  714. package/src/duckdb/src/include/duckdb/common/operator/cast_operators.hpp +16 -0
  715. package/src/duckdb/src/include/duckdb/common/optional_idx.hpp +4 -0
  716. package/src/duckdb/src/include/duckdb/common/platform.hpp +34 -3
  717. package/src/duckdb/src/include/duckdb/common/radix_partitioning.hpp +10 -13
  718. package/src/duckdb/src/include/duckdb/common/random_engine.hpp +8 -3
  719. package/src/duckdb/src/include/duckdb/common/row_operations/row_operations.hpp +0 -2
  720. package/src/duckdb/src/include/duckdb/common/serializer/buffered_file_writer.hpp +1 -1
  721. package/src/duckdb/src/include/duckdb/common/serializer/memory_stream.hpp +7 -0
  722. package/src/duckdb/src/include/duckdb/common/serializer/serialization_traits.hpp +1 -0
  723. package/src/duckdb/src/include/duckdb/common/serializer/serializer.hpp +1 -0
  724. package/src/duckdb/src/include/duckdb/common/stacktrace.hpp +25 -0
  725. package/src/duckdb/src/include/duckdb/common/string_util.hpp +30 -2
  726. package/src/duckdb/src/include/duckdb/common/tree_renderer/graphviz_tree_renderer.hpp +1 -1
  727. package/src/duckdb/src/include/duckdb/common/tree_renderer/html_tree_renderer.hpp +1 -1
  728. package/src/duckdb/src/include/duckdb/common/tree_renderer/json_tree_renderer.hpp +1 -1
  729. package/src/duckdb/src/include/duckdb/common/tree_renderer/text_tree_renderer.hpp +3 -2
  730. package/src/duckdb/src/include/duckdb/common/tree_renderer.hpp +2 -0
  731. package/src/duckdb/src/include/duckdb/common/type_util.hpp +8 -0
  732. package/src/duckdb/src/include/duckdb/common/types/column/column_data_allocator.hpp +8 -0
  733. package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection.hpp +13 -2
  734. package/src/duckdb/src/include/duckdb/common/types/column/column_data_collection_segment.hpp +2 -1
  735. package/src/duckdb/src/include/duckdb/common/types/conflict_manager.hpp +21 -4
  736. package/src/duckdb/src/include/duckdb/common/types/data_chunk.hpp +4 -1
  737. package/src/duckdb/src/include/duckdb/common/types/date.hpp +9 -4
  738. package/src/duckdb/src/include/duckdb/common/types/date_lookup_cache.hpp +1 -1
  739. package/src/duckdb/src/include/duckdb/common/types/interval.hpp +58 -10
  740. package/src/duckdb/src/include/duckdb/common/types/row/partitioned_tuple_data.hpp +0 -4
  741. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_allocator.hpp +4 -0
  742. package/src/duckdb/src/include/duckdb/common/types/row/tuple_data_collection.hpp +4 -0
  743. package/src/duckdb/src/include/duckdb/common/types/string_type.hpp +10 -0
  744. package/src/duckdb/src/include/duckdb/common/types/timestamp.hpp +43 -16
  745. package/src/duckdb/src/include/duckdb/common/types/uuid.hpp +3 -1
  746. package/src/duckdb/src/include/duckdb/common/types/validity_mask.hpp +63 -21
  747. package/src/duckdb/src/include/duckdb/common/types/value.hpp +62 -16
  748. package/src/duckdb/src/include/duckdb/common/types/varint.hpp +13 -0
  749. package/src/duckdb/src/include/duckdb/common/types/vector.hpp +34 -7
  750. package/src/duckdb/src/include/duckdb/common/types/vector_buffer.hpp +15 -0
  751. package/src/duckdb/src/include/duckdb/common/types.hpp +12 -7
  752. package/src/duckdb/src/include/duckdb/common/uhugeint.hpp +10 -0
  753. package/src/duckdb/src/include/duckdb/common/vector_operations/aggregate_executor.hpp +12 -13
  754. package/src/duckdb/src/include/duckdb/common/vector_operations/binary_executor.hpp +27 -0
  755. package/src/duckdb/src/include/duckdb/common/vector_operations/unary_executor.hpp +111 -4
  756. package/src/duckdb/src/include/duckdb/common/vector_operations/vector_operations.hpp +0 -1
  757. package/src/duckdb/src/include/duckdb/execution/adaptive_filter.hpp +2 -0
  758. package/src/duckdb/src/include/duckdb/execution/aggregate_hashtable.hpp +48 -10
  759. package/src/duckdb/src/include/duckdb/execution/executor.hpp +2 -1
  760. package/src/duckdb/src/include/duckdb/execution/expression_executor.hpp +0 -1
  761. package/src/duckdb/src/include/duckdb/execution/ht_entry.hpp +25 -27
  762. package/src/duckdb/src/include/duckdb/execution/index/art/art.hpp +28 -18
  763. package/src/duckdb/src/include/duckdb/execution/index/art/node.hpp +1 -0
  764. package/src/duckdb/src/include/duckdb/execution/index/art/prefix.hpp +2 -2
  765. package/src/duckdb/src/include/duckdb/execution/index/bound_index.hpp +23 -16
  766. package/src/duckdb/src/include/duckdb/execution/index/fixed_size_allocator.hpp +4 -0
  767. package/src/duckdb/src/include/duckdb/execution/index/fixed_size_buffer.hpp +2 -2
  768. package/src/duckdb/src/include/duckdb/execution/join_hashtable.hpp +25 -16
  769. package/src/duckdb/src/include/duckdb/execution/merge_sort_tree.hpp +15 -10
  770. package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_hash_aggregate.hpp +1 -1
  771. package/src/duckdb/src/include/duckdb/execution/operator/{persistent/physical_fixed_batch_copy.hpp → aggregate/physical_partitioned_aggregate.hpp} +25 -27
  772. package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_ungrouped_aggregate.hpp +1 -2
  773. package/src/duckdb/src/include/duckdb/execution/operator/aggregate/physical_window.hpp +5 -4
  774. package/src/duckdb/src/include/duckdb/execution/operator/aggregate/ungrouped_aggregate_state.hpp +21 -1
  775. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/base_scanner.hpp +38 -9
  776. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_buffer.hpp +8 -9
  777. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_buffer_manager.hpp +7 -1
  778. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_error.hpp +29 -23
  779. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_file_handle.hpp +15 -13
  780. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_file_scanner.hpp +13 -5
  781. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_option.hpp +2 -1
  782. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_reader_options.hpp +24 -10
  783. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_schema.hpp +36 -1
  784. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_state.hpp +21 -13
  785. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_state_machine.hpp +52 -22
  786. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_state_machine_cache.hpp +6 -6
  787. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_validator.hpp +58 -0
  788. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/encode/csv_encoder.hpp +62 -0
  789. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/global_csv_state.hpp +6 -3
  790. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/scanner_boundary.hpp +16 -6
  791. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/sniffer/csv_sniffer.hpp +9 -4
  792. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/state_machine_options.hpp +8 -4
  793. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/string_value_scanner.hpp +55 -10
  794. package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_batch_collector.hpp +2 -2
  795. package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_buffered_batch_collector.hpp +2 -2
  796. package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_limit.hpp +2 -2
  797. package/src/duckdb/src/include/duckdb/execution/operator/helper/physical_reservoir_sample.hpp +1 -1
  798. package/src/duckdb/src/include/duckdb/execution/operator/join/join_filter_pushdown.hpp +28 -7
  799. package/src/duckdb/src/include/duckdb/execution/operator/join/perfect_hash_join_executor.hpp +6 -9
  800. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_hash_join.hpp +17 -16
  801. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_iejoin.hpp +1 -1
  802. package/src/duckdb/src/include/duckdb/execution/operator/order/physical_order.hpp +7 -3
  803. package/src/duckdb/src/include/duckdb/execution/operator/order/physical_top_n.hpp +5 -1
  804. package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_batch_copy_to_file.hpp +2 -2
  805. package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_batch_insert.hpp +2 -2
  806. package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_insert.hpp +55 -4
  807. package/src/duckdb/src/include/duckdb/execution/operator/persistent/physical_update.hpp +2 -0
  808. package/src/duckdb/src/include/duckdb/execution/operator/projection/physical_tableinout_function.hpp +2 -2
  809. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_positional_scan.hpp +2 -1
  810. package/src/duckdb/src/include/duckdb/execution/operator/scan/physical_table_scan.hpp +10 -9
  811. package/src/duckdb/src/include/duckdb/execution/operator/schema/physical_create_art_index.hpp +16 -13
  812. package/src/duckdb/src/include/duckdb/execution/operator/set/physical_cte.hpp +0 -4
  813. package/src/duckdb/src/include/duckdb/execution/partition_info.hpp +79 -0
  814. package/src/duckdb/src/include/duckdb/execution/physical_operator.hpp +20 -9
  815. package/src/duckdb/src/include/duckdb/execution/physical_operator_states.hpp +1 -11
  816. package/src/duckdb/src/include/duckdb/execution/physical_plan_generator.hpp +0 -2
  817. package/src/duckdb/src/include/duckdb/execution/progress_data.hpp +58 -0
  818. package/src/duckdb/src/include/duckdb/execution/radix_partitioned_hashtable.hpp +2 -1
  819. package/src/duckdb/src/include/duckdb/execution/reservoir_sample.hpp +160 -31
  820. package/src/duckdb/src/include/duckdb/function/aggregate/distributive_function_utils.hpp +31 -0
  821. package/src/duckdb/src/include/duckdb/function/aggregate/distributive_functions.hpp +61 -10
  822. package/src/duckdb/src/include/duckdb/{core_functions → function}/aggregate/minmax_n_helpers.hpp +1 -1
  823. package/src/duckdb/src/include/duckdb/{core_functions → function}/aggregate/sort_key_helpers.hpp +2 -2
  824. package/src/duckdb/src/include/duckdb/function/aggregate_function.hpp +47 -27
  825. package/src/duckdb/src/include/duckdb/function/aggregate_state.hpp +2 -0
  826. package/src/duckdb/src/include/duckdb/function/built_in_functions.hpp +3 -10
  827. package/src/duckdb/src/include/duckdb/function/cast/bound_cast_data.hpp +13 -6
  828. package/src/duckdb/src/include/duckdb/function/compression/compression.hpp +15 -0
  829. package/src/duckdb/src/include/duckdb/function/compression_function.hpp +29 -6
  830. package/src/duckdb/src/include/duckdb/{core_functions → function}/create_sort_key.hpp +4 -1
  831. package/src/duckdb/src/include/duckdb/function/encoding_function.hpp +78 -0
  832. package/src/duckdb/src/include/duckdb/function/function.hpp +22 -1
  833. package/src/duckdb/src/include/duckdb/function/function_binder.hpp +3 -0
  834. package/src/duckdb/src/include/duckdb/function/function_list.hpp +39 -0
  835. package/src/duckdb/src/include/duckdb/function/function_set.hpp +13 -7
  836. package/src/duckdb/src/include/duckdb/{core_functions → function}/lambda_functions.hpp +1 -1
  837. package/src/duckdb/src/include/duckdb/function/partition_stats.hpp +36 -0
  838. package/src/duckdb/src/include/duckdb/function/register_function_list_helper.hpp +69 -0
  839. package/src/duckdb/src/include/duckdb/function/scalar/compressed_materialization_functions.hpp +154 -23
  840. package/src/duckdb/src/include/duckdb/function/scalar/compressed_materialization_utils.hpp +45 -0
  841. package/src/duckdb/src/include/duckdb/function/scalar/date_functions.hpp +45 -0
  842. package/src/duckdb/src/include/duckdb/function/scalar/generic_common.hpp +36 -0
  843. package/src/duckdb/src/include/duckdb/function/scalar/generic_functions.hpp +32 -23
  844. package/src/duckdb/src/include/duckdb/function/scalar/list/contains_or_position.hpp +1 -1
  845. package/src/duckdb/src/include/duckdb/function/scalar/list_functions.hpp +156 -0
  846. package/src/duckdb/src/include/duckdb/function/scalar/map_functions.hpp +27 -0
  847. package/src/duckdb/src/include/duckdb/function/scalar/nested_functions.hpp +4 -45
  848. package/src/duckdb/src/include/duckdb/function/scalar/operator_functions.hpp +102 -0
  849. package/src/duckdb/src/include/duckdb/function/scalar/operators.hpp +2 -16
  850. package/src/duckdb/src/include/duckdb/function/scalar/sequence_functions.hpp +16 -25
  851. package/src/duckdb/src/include/duckdb/function/scalar/sequence_utils.hpp +38 -0
  852. package/src/duckdb/src/include/duckdb/function/scalar/strftime_format.hpp +1 -0
  853. package/src/duckdb/src/include/duckdb/function/scalar/string_common.hpp +49 -0
  854. package/src/duckdb/src/include/duckdb/function/scalar/string_functions.hpp +401 -76
  855. package/src/duckdb/src/include/duckdb/function/scalar/struct_functions.hpp +63 -0
  856. package/src/duckdb/src/include/duckdb/function/scalar/struct_utils.hpp +33 -0
  857. package/src/duckdb/src/include/duckdb/function/scalar/system_functions.hpp +45 -0
  858. package/src/duckdb/src/include/duckdb/function/scalar_function.hpp +17 -8
  859. package/src/duckdb/src/include/duckdb/function/table/arrow/arrow_duck_schema.hpp +59 -6
  860. package/src/duckdb/src/include/duckdb/function/table/arrow/arrow_type_info.hpp +12 -9
  861. package/src/duckdb/src/include/duckdb/function/table/arrow/enum/arrow_type_info_type.hpp +2 -0
  862. package/src/duckdb/src/include/duckdb/function/table/arrow.hpp +18 -13
  863. package/src/duckdb/src/include/duckdb/function/table/read_csv.hpp +7 -4
  864. package/src/duckdb/src/include/duckdb/function/table/system_functions.hpp +14 -0
  865. package/src/duckdb/src/include/duckdb/function/table/table_scan.hpp +15 -10
  866. package/src/duckdb/src/include/duckdb/function/table_function.hpp +94 -18
  867. package/src/duckdb/src/include/duckdb/{core_functions → function}/to_interval.hpp +1 -1
  868. package/src/duckdb/src/include/duckdb/function/window/window_aggregate_function.hpp +44 -0
  869. package/src/duckdb/src/include/duckdb/function/window/window_aggregate_states.hpp +56 -0
  870. package/src/duckdb/src/include/duckdb/function/window/window_aggregator.hpp +194 -0
  871. package/src/duckdb/src/include/duckdb/function/window/window_boundaries_state.hpp +153 -0
  872. package/src/duckdb/src/include/duckdb/function/window/window_collection.hpp +146 -0
  873. package/src/duckdb/src/include/duckdb/function/window/window_constant_aggregator.hpp +38 -0
  874. package/src/duckdb/src/include/duckdb/function/window/window_custom_aggregator.hpp +32 -0
  875. package/src/duckdb/src/include/duckdb/function/window/window_distinct_aggregator.hpp +39 -0
  876. package/src/duckdb/src/include/duckdb/function/window/window_executor.hpp +122 -0
  877. package/src/duckdb/src/include/duckdb/function/window/window_index_tree.hpp +42 -0
  878. package/src/duckdb/src/include/duckdb/function/window/window_merge_sort_tree.hpp +108 -0
  879. package/src/duckdb/src/include/duckdb/function/window/window_naive_aggregator.hpp +33 -0
  880. package/src/duckdb/src/include/duckdb/function/window/window_rank_function.hpp +63 -0
  881. package/src/duckdb/src/include/duckdb/function/window/window_rownumber_function.hpp +43 -0
  882. package/src/duckdb/src/include/duckdb/function/window/window_segment_tree.hpp +31 -0
  883. package/src/duckdb/src/include/duckdb/function/window/window_shared_expressions.hpp +76 -0
  884. package/src/duckdb/src/include/duckdb/function/window/window_token_tree.hpp +46 -0
  885. package/src/duckdb/src/include/duckdb/function/window/window_value_function.hpp +79 -0
  886. package/src/duckdb/src/include/duckdb/logging/http_logger.hpp +2 -0
  887. package/src/duckdb/src/include/duckdb/logging/log_manager.hpp +81 -0
  888. package/src/duckdb/src/include/duckdb/logging/log_storage.hpp +127 -0
  889. package/src/duckdb/src/include/duckdb/logging/logger.hpp +287 -0
  890. package/src/duckdb/src/include/duckdb/logging/logging.hpp +83 -0
  891. package/src/duckdb/src/include/duckdb/main/appender.hpp +41 -18
  892. package/src/duckdb/src/include/duckdb/main/attached_database.hpp +6 -3
  893. package/src/duckdb/src/include/duckdb/main/capi/capi_internal.hpp +7 -2
  894. package/src/duckdb/src/include/duckdb/main/capi/extension_api.hpp +317 -231
  895. package/src/duckdb/src/include/duckdb/main/client_config.hpp +17 -1
  896. package/src/duckdb/src/include/duckdb/main/client_context.hpp +28 -6
  897. package/src/duckdb/src/include/duckdb/main/client_context_file_opener.hpp +1 -0
  898. package/src/duckdb/src/include/duckdb/main/client_context_wrapper.hpp +5 -0
  899. package/src/duckdb/src/include/duckdb/main/client_data.hpp +3 -2
  900. package/src/duckdb/src/include/duckdb/main/client_properties.hpp +8 -3
  901. package/src/duckdb/src/include/duckdb/main/config.hpp +52 -8
  902. package/src/duckdb/src/include/duckdb/main/connection.hpp +18 -3
  903. package/src/duckdb/src/include/duckdb/main/database.hpp +8 -7
  904. package/src/duckdb/src/include/duckdb/main/database_file_opener.hpp +5 -1
  905. package/src/duckdb/src/include/duckdb/main/database_manager.hpp +3 -0
  906. package/src/duckdb/src/include/duckdb/main/db_instance_cache.hpp +1 -0
  907. package/src/duckdb/src/include/duckdb/main/extension.hpp +8 -2
  908. package/src/duckdb/src/include/duckdb/main/extension_entries.hpp +548 -9
  909. package/src/duckdb/src/include/duckdb/main/extension_helper.hpp +18 -0
  910. package/src/duckdb/src/include/duckdb/main/extension_util.hpp +12 -7
  911. package/src/duckdb/src/include/duckdb/main/prepared_statement.hpp +3 -3
  912. package/src/duckdb/src/include/duckdb/main/profiling_info.hpp +2 -2
  913. package/src/duckdb/src/include/duckdb/main/query_profiler.hpp +8 -4
  914. package/src/duckdb/src/include/duckdb/main/relation/create_table_relation.hpp +3 -1
  915. package/src/duckdb/src/include/duckdb/main/relation/delete_relation.hpp +2 -2
  916. package/src/duckdb/src/include/duckdb/main/relation/subquery_relation.hpp +1 -4
  917. package/src/duckdb/src/include/duckdb/main/relation/table_function_relation.hpp +3 -1
  918. package/src/duckdb/src/include/duckdb/main/relation/table_relation.hpp +3 -0
  919. package/src/duckdb/src/include/duckdb/main/relation/update_relation.hpp +3 -2
  920. package/src/duckdb/src/include/duckdb/main/relation/value_relation.hpp +7 -0
  921. package/src/duckdb/src/include/duckdb/main/relation/view_relation.hpp +1 -0
  922. package/src/duckdb/src/include/duckdb/main/relation/write_parquet_relation.hpp +1 -1
  923. package/src/duckdb/src/include/duckdb/main/relation.hpp +45 -9
  924. package/src/duckdb/src/include/duckdb/main/secret/secret_storage.hpp +20 -22
  925. package/src/duckdb/src/include/duckdb/main/settings.hpp +613 -378
  926. package/src/duckdb/src/include/duckdb/main/table_description.hpp +14 -4
  927. package/src/duckdb/src/include/duckdb/optimizer/build_probe_side_optimizer.hpp +1 -3
  928. package/src/duckdb/src/include/duckdb/optimizer/column_lifetime_analyzer.hpp +14 -7
  929. package/src/duckdb/src/include/duckdb/optimizer/common_aggregate_optimizer.hpp +2 -2
  930. package/src/duckdb/src/include/duckdb/optimizer/empty_result_pullup.hpp +27 -0
  931. package/src/duckdb/src/include/duckdb/optimizer/expression_heuristics.hpp +1 -1
  932. package/src/duckdb/src/include/duckdb/optimizer/filter_combiner.hpp +6 -1
  933. package/src/duckdb/src/include/duckdb/optimizer/filter_pushdown.hpp +2 -0
  934. package/src/duckdb/src/include/duckdb/optimizer/in_clause_rewriter.hpp +3 -0
  935. package/src/duckdb/src/include/duckdb/optimizer/join_filter_pushdown_optimizer.hpp +5 -0
  936. package/src/duckdb/src/include/duckdb/optimizer/join_order/plan_enumerator.hpp +2 -0
  937. package/src/duckdb/src/include/duckdb/optimizer/join_order/relation_statistics_helper.hpp +2 -2
  938. package/src/duckdb/src/include/duckdb/optimizer/late_materialization.hpp +45 -0
  939. package/src/duckdb/src/include/duckdb/optimizer/matcher/expression_matcher.hpp +23 -0
  940. package/src/duckdb/src/include/duckdb/optimizer/matcher/type_matcher.hpp +18 -0
  941. package/src/duckdb/src/include/duckdb/optimizer/optimizer.hpp +9 -0
  942. package/src/duckdb/src/include/duckdb/optimizer/remove_unused_columns.hpp +33 -11
  943. package/src/duckdb/src/include/duckdb/optimizer/rule/distinct_aggregate_optimizer.hpp +34 -0
  944. package/src/duckdb/src/include/duckdb/optimizer/sampling_pushdown.hpp +25 -0
  945. package/src/duckdb/src/include/duckdb/optimizer/statistics_propagator.hpp +3 -1
  946. package/src/duckdb/src/include/duckdb/optimizer/sum_rewriter.hpp +37 -0
  947. package/src/duckdb/src/include/duckdb/optimizer/topn_optimizer.hpp +4 -0
  948. package/src/duckdb/src/include/duckdb/parallel/event.hpp +3 -0
  949. package/src/duckdb/src/include/duckdb/parallel/pipeline.hpp +1 -1
  950. package/src/duckdb/src/include/duckdb/parallel/pipeline_executor.hpp +26 -8
  951. package/src/duckdb/src/include/duckdb/parallel/thread_context.hpp +3 -0
  952. package/src/duckdb/src/include/duckdb/parser/base_expression.hpp +51 -3
  953. package/src/duckdb/src/include/duckdb/parser/constraints/unique_constraint.hpp +28 -44
  954. package/src/duckdb/src/include/duckdb/parser/expression/columnref_expression.hpp +3 -0
  955. package/src/duckdb/src/include/duckdb/parser/expression/comparison_expression.hpp +2 -2
  956. package/src/duckdb/src/include/duckdb/parser/expression/conjunction_expression.hpp +1 -1
  957. package/src/duckdb/src/include/duckdb/parser/expression/function_expression.hpp +2 -2
  958. package/src/duckdb/src/include/duckdb/parser/expression/operator_expression.hpp +6 -6
  959. package/src/duckdb/src/include/duckdb/parser/expression/star_expression.hpp +11 -1
  960. package/src/duckdb/src/include/duckdb/parser/expression/window_expression.hpp +12 -0
  961. package/src/duckdb/src/include/duckdb/parser/parsed_data/alter_info.hpp +1 -0
  962. package/src/duckdb/src/include/duckdb/parser/parsed_data/alter_scalar_function_info.hpp +3 -2
  963. package/src/duckdb/src/include/duckdb/parser/parsed_data/alter_table_info.hpp +22 -1
  964. package/src/duckdb/src/include/duckdb/parser/parsed_data/attach_info.hpp +3 -4
  965. package/src/duckdb/src/include/duckdb/parser/parsed_data/comment_on_column_info.hpp +1 -1
  966. package/src/duckdb/src/include/duckdb/parser/parsed_data/create_function_info.hpp +16 -12
  967. package/src/duckdb/src/include/duckdb/parser/parsed_data/create_index_info.hpp +3 -3
  968. package/src/duckdb/src/include/duckdb/parser/parsed_data/create_type_info.hpp +5 -5
  969. package/src/duckdb/src/include/duckdb/parser/parsed_data/sample_options.hpp +12 -3
  970. package/src/duckdb/src/include/duckdb/parser/parser.hpp +3 -0
  971. package/src/duckdb/src/include/duckdb/parser/qualified_name.hpp +17 -57
  972. package/src/duckdb/src/include/duckdb/parser/qualified_name_set.hpp +19 -3
  973. package/src/duckdb/src/include/duckdb/parser/simplified_token.hpp +2 -1
  974. package/src/duckdb/src/include/duckdb/parser/tableref/basetableref.hpp +12 -9
  975. package/src/duckdb/src/include/duckdb/parser/tokens.hpp +1 -1
  976. package/src/duckdb/src/include/duckdb/parser/transformer.hpp +2 -2
  977. package/src/duckdb/src/include/duckdb/planner/bind_context.hpp +45 -28
  978. package/src/duckdb/src/include/duckdb/planner/binder.hpp +23 -11
  979. package/src/duckdb/src/include/duckdb/planner/binding_alias.hpp +44 -0
  980. package/src/duckdb/src/include/duckdb/planner/bound_result_modifier.hpp +1 -0
  981. package/src/duckdb/src/include/duckdb/planner/collation_binding.hpp +4 -3
  982. package/src/duckdb/src/include/duckdb/planner/constraints/bound_unique_constraint.hpp +11 -10
  983. package/src/duckdb/src/include/duckdb/planner/expression/bound_cast_expression.hpp +2 -0
  984. package/src/duckdb/src/include/duckdb/planner/expression/bound_function_expression.hpp +1 -0
  985. package/src/duckdb/src/include/duckdb/planner/expression/bound_subquery_expression.hpp +4 -4
  986. package/src/duckdb/src/include/duckdb/planner/expression/bound_window_expression.hpp +6 -0
  987. package/src/duckdb/src/include/duckdb/planner/expression.hpp +2 -0
  988. package/src/duckdb/src/include/duckdb/planner/expression_binder/having_binder.hpp +2 -0
  989. package/src/duckdb/src/include/duckdb/planner/expression_binder/index_binder.hpp +9 -4
  990. package/src/duckdb/src/include/duckdb/planner/expression_binder.hpp +8 -2
  991. package/src/duckdb/src/include/duckdb/planner/filter/conjunction_filter.hpp +1 -2
  992. package/src/duckdb/src/include/duckdb/planner/filter/dynamic_filter.hpp +48 -0
  993. package/src/duckdb/src/include/duckdb/planner/filter/in_filter.hpp +37 -0
  994. package/src/duckdb/src/include/duckdb/planner/filter/optional_filter.hpp +35 -0
  995. package/src/duckdb/src/include/duckdb/planner/logical_operator.hpp +4 -0
  996. package/src/duckdb/src/include/duckdb/planner/logical_operator_visitor.hpp +3 -0
  997. package/src/duckdb/src/include/duckdb/planner/operator/logical_comparison_join.hpp +2 -0
  998. package/src/duckdb/src/include/duckdb/planner/operator/logical_create_index.hpp +9 -9
  999. package/src/duckdb/src/include/duckdb/planner/operator/logical_filter.hpp +4 -0
  1000. package/src/duckdb/src/include/duckdb/planner/operator/logical_get.hpp +16 -7
  1001. package/src/duckdb/src/include/duckdb/planner/operator/logical_insert.hpp +2 -0
  1002. package/src/duckdb/src/include/duckdb/planner/operator/logical_join.hpp +4 -0
  1003. package/src/duckdb/src/include/duckdb/planner/operator/logical_order.hpp +5 -1
  1004. package/src/duckdb/src/include/duckdb/planner/operator/logical_top_n.hpp +5 -3
  1005. package/src/duckdb/src/include/duckdb/planner/table_binding.hpp +14 -6
  1006. package/src/duckdb/src/include/duckdb/planner/table_filter.hpp +12 -8
  1007. package/src/duckdb/src/include/duckdb/storage/arena_allocator.hpp +1 -0
  1008. package/src/duckdb/src/include/duckdb/storage/block_manager.hpp +3 -0
  1009. package/src/duckdb/src/include/duckdb/storage/buffer/block_handle.hpp +82 -26
  1010. package/src/duckdb/src/include/duckdb/storage/buffer/buffer_handle.hpp +1 -1
  1011. package/src/duckdb/src/include/duckdb/storage/buffer/buffer_pool.hpp +10 -3
  1012. package/src/duckdb/src/include/duckdb/storage/buffer_manager.hpp +2 -1
  1013. package/src/duckdb/src/include/duckdb/storage/checkpoint/string_checkpoint_state.hpp +4 -13
  1014. package/src/duckdb/src/include/duckdb/storage/compression/alp/alp_compress.hpp +14 -15
  1015. package/src/duckdb/src/include/duckdb/storage/compression/alp/alp_constants.hpp +1 -1
  1016. package/src/duckdb/src/include/duckdb/storage/compression/alprd/alprd_compress.hpp +13 -15
  1017. package/src/duckdb/src/include/duckdb/storage/compression/chimp/chimp_compress.hpp +1 -1
  1018. package/src/duckdb/src/include/duckdb/storage/compression/dictionary/analyze.hpp +46 -0
  1019. package/src/duckdb/src/include/duckdb/storage/compression/dictionary/common.hpp +60 -0
  1020. package/src/duckdb/src/include/duckdb/storage/compression/dictionary/compression.hpp +61 -0
  1021. package/src/duckdb/src/include/duckdb/storage/compression/dictionary/decompression.hpp +50 -0
  1022. package/src/duckdb/src/include/duckdb/storage/compression/empty_validity.hpp +100 -0
  1023. package/src/duckdb/src/include/duckdb/storage/compression/patas/patas_compress.hpp +1 -1
  1024. package/src/duckdb/src/include/duckdb/storage/compression/roaring/appender.hpp +150 -0
  1025. package/src/duckdb/src/include/duckdb/storage/compression/roaring/roaring.hpp +618 -0
  1026. package/src/duckdb/src/include/duckdb/storage/data_table.hpp +53 -31
  1027. package/src/duckdb/src/include/duckdb/storage/index.hpp +2 -3
  1028. package/src/duckdb/src/include/duckdb/storage/object_cache.hpp +0 -1
  1029. package/src/duckdb/src/include/duckdb/storage/segment/uncompressed.hpp +4 -1
  1030. package/src/duckdb/src/include/duckdb/storage/standard_buffer_manager.hpp +3 -3
  1031. package/src/duckdb/src/include/duckdb/storage/statistics/column_statistics.hpp +1 -1
  1032. package/src/duckdb/src/include/duckdb/storage/statistics/distinct_statistics.hpp +5 -4
  1033. package/src/duckdb/src/include/duckdb/storage/statistics/numeric_stats.hpp +16 -1
  1034. package/src/duckdb/src/include/duckdb/storage/statistics/string_stats.hpp +2 -1
  1035. package/src/duckdb/src/include/duckdb/storage/storage_index.hpp +70 -0
  1036. package/src/duckdb/src/include/duckdb/storage/storage_info.hpp +5 -7
  1037. package/src/duckdb/src/include/duckdb/storage/storage_manager.hpp +4 -3
  1038. package/src/duckdb/src/include/duckdb/storage/storage_options.hpp +23 -0
  1039. package/src/duckdb/src/include/duckdb/storage/string_uncompressed.hpp +34 -6
  1040. package/src/duckdb/src/include/duckdb/storage/table/append_state.hpp +2 -0
  1041. package/src/duckdb/src/include/duckdb/storage/table/array_column_data.hpp +2 -2
  1042. package/src/duckdb/src/include/duckdb/storage/table/column_checkpoint_state.hpp +2 -1
  1043. package/src/duckdb/src/include/duckdb/storage/table/column_data.hpp +39 -10
  1044. package/src/duckdb/src/include/duckdb/storage/table/column_data_checkpointer.hpp +56 -14
  1045. package/src/duckdb/src/include/duckdb/storage/table/column_segment.hpp +35 -29
  1046. package/src/duckdb/src/include/duckdb/storage/table/delete_state.hpp +1 -1
  1047. package/src/duckdb/src/include/duckdb/storage/table/list_column_data.hpp +1 -1
  1048. package/src/duckdb/src/include/duckdb/storage/table/row_group.hpp +7 -1
  1049. package/src/duckdb/src/include/duckdb/storage/table/row_group_collection.hpp +19 -6
  1050. package/src/duckdb/src/include/duckdb/storage/table/row_version_manager.hpp +2 -1
  1051. package/src/duckdb/src/include/duckdb/storage/table/scan_state.hpp +29 -6
  1052. package/src/duckdb/src/include/duckdb/storage/table/segment_tree.hpp +10 -10
  1053. package/src/duckdb/src/include/duckdb/storage/table/standard_column_data.hpp +5 -0
  1054. package/src/duckdb/src/include/duckdb/storage/table/table_index_list.hpp +26 -19
  1055. package/src/duckdb/src/include/duckdb/storage/table/table_statistics.hpp +8 -1
  1056. package/src/duckdb/src/include/duckdb/storage/table/update_segment.hpp +16 -14
  1057. package/src/duckdb/src/include/duckdb/storage/table/validity_column_data.hpp +2 -0
  1058. package/src/duckdb/src/include/duckdb/storage/table_io_manager.hpp +3 -0
  1059. package/src/duckdb/src/include/duckdb/storage/table_storage_info.hpp +1 -0
  1060. package/src/duckdb/src/include/duckdb/storage/temporary_file_manager.hpp +228 -61
  1061. package/src/duckdb/src/include/duckdb/storage/write_ahead_log.hpp +14 -10
  1062. package/src/duckdb/src/include/duckdb/transaction/commit_state.hpp +3 -1
  1063. package/src/duckdb/src/include/duckdb/transaction/duck_transaction.hpp +3 -2
  1064. package/src/duckdb/src/include/duckdb/transaction/duck_transaction_manager.hpp +1 -0
  1065. package/src/duckdb/src/include/duckdb/transaction/local_storage.hpp +19 -17
  1066. package/src/duckdb/src/include/duckdb/transaction/rollback_state.hpp +5 -2
  1067. package/src/duckdb/src/include/duckdb/transaction/transaction.hpp +1 -2
  1068. package/src/duckdb/src/include/duckdb/transaction/undo_buffer.hpp +13 -8
  1069. package/src/duckdb/src/include/duckdb/transaction/undo_buffer_allocator.hpp +79 -0
  1070. package/src/duckdb/src/include/duckdb/transaction/update_info.hpp +43 -13
  1071. package/src/duckdb/src/include/duckdb/transaction/wal_write_state.hpp +4 -1
  1072. package/src/duckdb/src/include/duckdb/verification/copied_statement_verifier.hpp +4 -2
  1073. package/src/duckdb/src/include/duckdb/verification/deserialized_statement_verifier.hpp +4 -2
  1074. package/src/duckdb/src/include/duckdb/verification/external_statement_verifier.hpp +4 -2
  1075. package/src/duckdb/src/include/duckdb/verification/fetch_row_verifier.hpp +4 -2
  1076. package/src/duckdb/src/include/duckdb/verification/no_operator_caching_verifier.hpp +4 -2
  1077. package/src/duckdb/src/include/duckdb/verification/parsed_statement_verifier.hpp +4 -2
  1078. package/src/duckdb/src/include/duckdb/verification/prepared_statement_verifier.hpp +7 -3
  1079. package/src/duckdb/src/include/duckdb/verification/statement_verifier.hpp +11 -5
  1080. package/src/duckdb/src/include/duckdb/verification/unoptimized_statement_verifier.hpp +4 -2
  1081. package/src/duckdb/src/include/duckdb.h +424 -41
  1082. package/src/duckdb/src/include/duckdb_extension.h +301 -195
  1083. package/src/duckdb/src/logging/log_manager.cpp +157 -0
  1084. package/src/duckdb/src/logging/log_storage.cpp +209 -0
  1085. package/src/duckdb/src/logging/logger.cpp +211 -0
  1086. package/src/duckdb/src/logging/logging.cpp +42 -0
  1087. package/src/duckdb/src/main/appender.cpp +187 -45
  1088. package/src/duckdb/src/main/attached_database.cpp +16 -8
  1089. package/src/duckdb/src/main/capi/appender-c.cpp +47 -4
  1090. package/src/duckdb/src/main/capi/arrow-c.cpp +9 -4
  1091. package/src/duckdb/src/main/capi/config-c.cpp +17 -4
  1092. package/src/duckdb/src/main/capi/datetime-c.cpp +15 -0
  1093. package/src/duckdb/src/main/capi/duckdb-c.cpp +54 -13
  1094. package/src/duckdb/src/main/capi/duckdb_value-c.cpp +212 -4
  1095. package/src/duckdb/src/main/capi/helper-c.cpp +3 -0
  1096. package/src/duckdb/src/main/capi/prepared-c.cpp +26 -7
  1097. package/src/duckdb/src/main/capi/replacement_scan-c.cpp +1 -1
  1098. package/src/duckdb/src/main/capi/result-c.cpp +3 -0
  1099. package/src/duckdb/src/main/capi/table_description-c.cpp +43 -10
  1100. package/src/duckdb/src/main/capi/threading-c.cpp +4 -4
  1101. package/src/duckdb/src/main/client_context.cpp +125 -51
  1102. package/src/duckdb/src/main/client_context_file_opener.cpp +4 -0
  1103. package/src/duckdb/src/main/client_context_wrapper.cpp +4 -0
  1104. package/src/duckdb/src/main/client_data.cpp +1 -1
  1105. package/src/duckdb/src/main/client_verify.cpp +39 -20
  1106. package/src/duckdb/src/main/config.cpp +266 -74
  1107. package/src/duckdb/src/main/connection.cpp +53 -13
  1108. package/src/duckdb/src/main/database.cpp +39 -18
  1109. package/src/duckdb/src/main/database_manager.cpp +12 -11
  1110. package/src/duckdb/src/main/db_instance_cache.cpp +14 -7
  1111. package/src/duckdb/src/main/extension/extension_helper.cpp +24 -23
  1112. package/src/duckdb/src/main/extension/extension_install.cpp +19 -7
  1113. package/src/duckdb/src/main/extension/extension_load.cpp +91 -41
  1114. package/src/duckdb/src/main/extension/extension_util.cpp +40 -19
  1115. package/src/duckdb/src/main/extension.cpp +20 -11
  1116. package/src/duckdb/src/main/profiling_info.cpp +19 -5
  1117. package/src/duckdb/src/main/query_profiler.cpp +135 -36
  1118. package/src/duckdb/src/main/query_result.cpp +2 -1
  1119. package/src/duckdb/src/main/relation/aggregate_relation.cpp +3 -3
  1120. package/src/duckdb/src/main/relation/create_table_relation.cpp +5 -4
  1121. package/src/duckdb/src/main/relation/create_view_relation.cpp +2 -2
  1122. package/src/duckdb/src/main/relation/cross_product_relation.cpp +2 -2
  1123. package/src/duckdb/src/main/relation/delete_relation.cpp +2 -2
  1124. package/src/duckdb/src/main/relation/delim_get_relation.cpp +1 -1
  1125. package/src/duckdb/src/main/relation/distinct_relation.cpp +1 -1
  1126. package/src/duckdb/src/main/relation/explain_relation.cpp +1 -1
  1127. package/src/duckdb/src/main/relation/filter_relation.cpp +1 -1
  1128. package/src/duckdb/src/main/relation/insert_relation.cpp +1 -1
  1129. package/src/duckdb/src/main/relation/join_relation.cpp +5 -5
  1130. package/src/duckdb/src/main/relation/order_relation.cpp +1 -1
  1131. package/src/duckdb/src/main/relation/projection_relation.cpp +3 -3
  1132. package/src/duckdb/src/main/relation/query_relation.cpp +1 -1
  1133. package/src/duckdb/src/main/relation/read_csv_relation.cpp +58 -20
  1134. package/src/duckdb/src/main/relation/setop_relation.cpp +2 -2
  1135. package/src/duckdb/src/main/relation/subquery_relation.cpp +3 -8
  1136. package/src/duckdb/src/main/relation/table_function_relation.cpp +10 -1
  1137. package/src/duckdb/src/main/relation/table_relation.cpp +19 -3
  1138. package/src/duckdb/src/main/relation/update_relation.cpp +2 -2
  1139. package/src/duckdb/src/main/relation/value_relation.cpp +42 -2
  1140. package/src/duckdb/src/main/relation/view_relation.cpp +8 -2
  1141. package/src/duckdb/src/main/relation/write_csv_relation.cpp +1 -1
  1142. package/src/duckdb/src/main/relation/write_parquet_relation.cpp +1 -1
  1143. package/src/duckdb/src/main/relation.cpp +49 -28
  1144. package/src/duckdb/src/main/secret/secret_manager.cpp +1 -1
  1145. package/src/duckdb/src/main/secret/secret_storage.cpp +6 -4
  1146. package/src/duckdb/src/main/settings/autogenerated_settings.cpp +1102 -0
  1147. package/src/duckdb/src/main/settings/custom_settings.cpp +1343 -0
  1148. package/src/duckdb/src/optimizer/build_probe_side_optimizer.cpp +60 -37
  1149. package/src/duckdb/src/optimizer/column_binding_replacer.cpp +1 -1
  1150. package/src/duckdb/src/optimizer/column_lifetime_analyzer.cpp +126 -72
  1151. package/src/duckdb/src/optimizer/common_aggregate_optimizer.cpp +22 -6
  1152. package/src/duckdb/src/optimizer/compressed_materialization/compress_aggregate.cpp +3 -3
  1153. package/src/duckdb/src/optimizer/compressed_materialization/compress_distinct.cpp +2 -2
  1154. package/src/duckdb/src/optimizer/compressed_materialization.cpp +3 -3
  1155. package/src/duckdb/src/optimizer/cse_optimizer.cpp +7 -7
  1156. package/src/duckdb/src/optimizer/deliminator.cpp +6 -5
  1157. package/src/duckdb/src/optimizer/empty_result_pullup.cpp +96 -0
  1158. package/src/duckdb/src/optimizer/expression_heuristics.cpp +11 -3
  1159. package/src/duckdb/src/optimizer/expression_rewriter.cpp +9 -2
  1160. package/src/duckdb/src/optimizer/filter_combiner.cpp +190 -88
  1161. package/src/duckdb/src/optimizer/filter_pushdown.cpp +6 -5
  1162. package/src/duckdb/src/optimizer/in_clause_rewriter.cpp +25 -9
  1163. package/src/duckdb/src/optimizer/join_filter_pushdown_optimizer.cpp +170 -72
  1164. package/src/duckdb/src/optimizer/join_order/cardinality_estimator.cpp +5 -4
  1165. package/src/duckdb/src/optimizer/join_order/plan_enumerator.cpp +3 -1
  1166. package/src/duckdb/src/optimizer/join_order/query_graph_manager.cpp +7 -7
  1167. package/src/duckdb/src/optimizer/join_order/relation_manager.cpp +15 -6
  1168. package/src/duckdb/src/optimizer/join_order/relation_statistics_helper.cpp +37 -22
  1169. package/src/duckdb/src/optimizer/late_materialization.cpp +414 -0
  1170. package/src/duckdb/src/optimizer/limit_pushdown.cpp +1 -0
  1171. package/src/duckdb/src/optimizer/matcher/expression_matcher.cpp +30 -2
  1172. package/src/duckdb/src/optimizer/optimizer.cpp +67 -7
  1173. package/src/duckdb/src/optimizer/pullup/pullup_filter.cpp +3 -3
  1174. package/src/duckdb/src/optimizer/pullup/pullup_projection.cpp +2 -2
  1175. package/src/duckdb/src/optimizer/pullup/pullup_set_operation.cpp +1 -1
  1176. package/src/duckdb/src/optimizer/pushdown/pushdown_aggregate.cpp +2 -2
  1177. package/src/duckdb/src/optimizer/pushdown/pushdown_filter.cpp +1 -1
  1178. package/src/duckdb/src/optimizer/pushdown/pushdown_left_join.cpp +1 -1
  1179. package/src/duckdb/src/optimizer/pushdown/pushdown_mark_join.cpp +3 -3
  1180. package/src/duckdb/src/optimizer/pushdown/pushdown_projection.cpp +5 -3
  1181. package/src/duckdb/src/optimizer/pushdown/pushdown_set_operation.cpp +1 -1
  1182. package/src/duckdb/src/optimizer/pushdown/pushdown_unnest.cpp +52 -0
  1183. package/src/duckdb/src/optimizer/pushdown/pushdown_window.cpp +2 -2
  1184. package/src/duckdb/src/optimizer/regex_range_filter.cpp +1 -1
  1185. package/src/duckdb/src/optimizer/remove_duplicate_groups.cpp +1 -1
  1186. package/src/duckdb/src/optimizer/remove_unused_columns.cpp +168 -38
  1187. package/src/duckdb/src/optimizer/rule/arithmetic_simplification.cpp +2 -1
  1188. package/src/duckdb/src/optimizer/rule/comparison_simplification.cpp +8 -5
  1189. package/src/duckdb/src/optimizer/rule/conjunction_simplification.cpp +2 -2
  1190. package/src/duckdb/src/optimizer/rule/constant_folding.cpp +2 -2
  1191. package/src/duckdb/src/optimizer/rule/distinct_aggregate_optimizer.cpp +65 -0
  1192. package/src/duckdb/src/optimizer/rule/distributivity.cpp +2 -2
  1193. package/src/duckdb/src/optimizer/rule/enum_comparison.cpp +2 -1
  1194. package/src/duckdb/src/optimizer/rule/equal_or_null_simplification.cpp +4 -3
  1195. package/src/duckdb/src/optimizer/rule/in_clause_simplification_rule.cpp +3 -3
  1196. package/src/duckdb/src/optimizer/rule/like_optimizations.cpp +3 -1
  1197. package/src/duckdb/src/optimizer/rule/move_constants.cpp +9 -9
  1198. package/src/duckdb/src/optimizer/rule/regex_optimizations.cpp +4 -3
  1199. package/src/duckdb/src/optimizer/rule/timestamp_comparison.cpp +1 -1
  1200. package/src/duckdb/src/optimizer/sampling_pushdown.cpp +24 -0
  1201. package/src/duckdb/src/optimizer/statistics/expression/propagate_comparison.cpp +1 -1
  1202. package/src/duckdb/src/optimizer/statistics/expression/propagate_conjunction.cpp +1 -1
  1203. package/src/duckdb/src/optimizer/statistics/expression/propagate_operator.cpp +1 -1
  1204. package/src/duckdb/src/optimizer/statistics/operator/propagate_aggregate.cpp +74 -0
  1205. package/src/duckdb/src/optimizer/statistics/operator/propagate_filter.cpp +10 -7
  1206. package/src/duckdb/src/optimizer/statistics/operator/propagate_get.cpp +3 -3
  1207. package/src/duckdb/src/optimizer/statistics/operator/propagate_join.cpp +3 -3
  1208. package/src/duckdb/src/optimizer/statistics/operator/propagate_window.cpp +3 -0
  1209. package/src/duckdb/src/optimizer/sum_rewriter.cpp +174 -0
  1210. package/src/duckdb/src/optimizer/topn_optimizer.cpp +71 -0
  1211. package/src/duckdb/src/optimizer/unnest_rewriter.cpp +5 -5
  1212. package/src/duckdb/src/parallel/event.cpp +4 -0
  1213. package/src/duckdb/src/parallel/executor.cpp +11 -29
  1214. package/src/duckdb/src/parallel/executor_task.cpp +8 -3
  1215. package/src/duckdb/src/parallel/pipeline.cpp +15 -8
  1216. package/src/duckdb/src/parallel/pipeline_executor.cpp +67 -43
  1217. package/src/duckdb/src/parallel/thread_context.cpp +12 -1
  1218. package/src/duckdb/src/parser/column_definition.cpp +3 -3
  1219. package/src/duckdb/src/parser/constraints/unique_constraint.cpp +72 -9
  1220. package/src/duckdb/src/parser/expression/columnref_expression.cpp +15 -3
  1221. package/src/duckdb/src/parser/expression/conjunction_expression.cpp +1 -1
  1222. package/src/duckdb/src/parser/expression/function_expression.cpp +1 -1
  1223. package/src/duckdb/src/parser/expression/lambda_expression.cpp +3 -3
  1224. package/src/duckdb/src/parser/expression/lambdaref_expression.cpp +1 -1
  1225. package/src/duckdb/src/parser/expression/star_expression.cpp +46 -2
  1226. package/src/duckdb/src/parser/expression/window_expression.cpp +24 -1
  1227. package/src/duckdb/src/parser/parsed_data/alter_info.cpp +26 -2
  1228. package/src/duckdb/src/parser/parsed_data/alter_scalar_function_info.cpp +5 -3
  1229. package/src/duckdb/src/parser/parsed_data/alter_table_info.cpp +29 -1
  1230. package/src/duckdb/src/parser/parsed_data/attach_info.cpp +6 -6
  1231. package/src/duckdb/src/parser/parsed_data/create_aggregate_function_info.cpp +1 -1
  1232. package/src/duckdb/src/parser/parsed_data/create_function_info.cpp +17 -0
  1233. package/src/duckdb/src/parser/parsed_data/create_index_info.cpp +16 -15
  1234. package/src/duckdb/src/parser/parsed_data/create_macro_info.cpp +1 -1
  1235. package/src/duckdb/src/parser/parsed_data/create_pragma_function_info.cpp +1 -1
  1236. package/src/duckdb/src/parser/parsed_data/create_scalar_function_info.cpp +3 -2
  1237. package/src/duckdb/src/parser/parsed_data/create_schema_info.cpp +1 -1
  1238. package/src/duckdb/src/parser/parsed_data/create_table_function_info.cpp +1 -1
  1239. package/src/duckdb/src/parser/parsed_data/create_table_info.cpp +1 -0
  1240. package/src/duckdb/src/parser/parsed_data/create_type_info.cpp +4 -4
  1241. package/src/duckdb/src/parser/parsed_data/load_info.cpp +1 -0
  1242. package/src/duckdb/src/parser/parsed_data/sample_options.cpp +31 -1
  1243. package/src/duckdb/src/parser/parsed_expression.cpp +1 -1
  1244. package/src/duckdb/src/parser/parsed_expression_iterator.cpp +4 -1
  1245. package/src/duckdb/src/parser/parser.cpp +129 -0
  1246. package/src/duckdb/src/parser/qualified_name.cpp +99 -0
  1247. package/src/duckdb/src/parser/query_error_context.cpp +35 -6
  1248. package/src/duckdb/src/parser/query_node/select_node.cpp +4 -4
  1249. package/src/duckdb/src/parser/statement/delete_statement.cpp +6 -1
  1250. package/src/duckdb/src/parser/statement/insert_statement.cpp +4 -3
  1251. package/src/duckdb/src/parser/statement/update_statement.cpp +6 -1
  1252. package/src/duckdb/src/parser/tableref/pivotref.cpp +2 -2
  1253. package/src/duckdb/src/parser/tableref.cpp +2 -2
  1254. package/src/duckdb/src/parser/transform/constraint/transform_constraint.cpp +16 -24
  1255. package/src/duckdb/src/parser/transform/expression/transform_array_access.cpp +1 -1
  1256. package/src/duckdb/src/parser/transform/expression/transform_bool_expr.cpp +5 -5
  1257. package/src/duckdb/src/parser/transform/expression/transform_columnref.cpp +61 -13
  1258. package/src/duckdb/src/parser/transform/expression/transform_constant.cpp +10 -4
  1259. package/src/duckdb/src/parser/transform/expression/transform_expression.cpp +2 -2
  1260. package/src/duckdb/src/parser/transform/expression/transform_function.cpp +30 -3
  1261. package/src/duckdb/src/parser/transform/expression/transform_operator.cpp +1 -1
  1262. package/src/duckdb/src/parser/transform/expression/transform_subquery.cpp +25 -6
  1263. package/src/duckdb/src/parser/transform/helpers/transform_groupby.cpp +1 -1
  1264. package/src/duckdb/src/parser/transform/helpers/transform_sample.cpp +10 -3
  1265. package/src/duckdb/src/parser/transform/helpers/transform_typename.cpp +4 -3
  1266. package/src/duckdb/src/parser/transform/statement/transform_alter_table.cpp +18 -3
  1267. package/src/duckdb/src/parser/transform/statement/transform_comment_on.cpp +1 -1
  1268. package/src/duckdb/src/parser/transform/statement/transform_copy.cpp +0 -1
  1269. package/src/duckdb/src/parser/transform/statement/transform_create_function.cpp +5 -5
  1270. package/src/duckdb/src/parser/transform/statement/transform_create_table.cpp +26 -12
  1271. package/src/duckdb/src/parser/transform/statement/transform_create_table_as.cpp +11 -3
  1272. package/src/duckdb/src/parser/transform/statement/transform_create_view.cpp +1 -1
  1273. package/src/duckdb/src/parser/transform/statement/transform_pivot_stmt.cpp +2 -0
  1274. package/src/duckdb/src/parser/transform/statement/transform_pragma.cpp +3 -3
  1275. package/src/duckdb/src/parser/transform/statement/transform_prepare.cpp +4 -4
  1276. package/src/duckdb/src/parser/transform/statement/transform_set.cpp +2 -2
  1277. package/src/duckdb/src/parser/transform/statement/transform_show.cpp +21 -3
  1278. package/src/duckdb/src/parser/transform/tableref/transform_pivot.cpp +8 -6
  1279. package/src/duckdb/src/parser/transformer.cpp +2 -2
  1280. package/src/duckdb/src/planner/bind_context.cpp +308 -136
  1281. package/src/duckdb/src/planner/binder/expression/bind_aggregate_expression.cpp +30 -31
  1282. package/src/duckdb/src/planner/binder/expression/bind_between_expression.cpp +4 -2
  1283. package/src/duckdb/src/planner/binder/expression/bind_columnref_expression.cpp +102 -94
  1284. package/src/duckdb/src/planner/binder/expression/bind_comparison_expression.cpp +7 -5
  1285. package/src/duckdb/src/planner/binder/expression/bind_conjunction_expression.cpp +1 -1
  1286. package/src/duckdb/src/planner/binder/expression/bind_function_expression.cpp +7 -7
  1287. package/src/duckdb/src/planner/binder/expression/bind_lambda.cpp +10 -10
  1288. package/src/duckdb/src/planner/binder/expression/bind_macro_expression.cpp +24 -6
  1289. package/src/duckdb/src/planner/binder/expression/bind_operator_expression.cpp +23 -15
  1290. package/src/duckdb/src/planner/binder/expression/bind_parameter_expression.cpp +1 -1
  1291. package/src/duckdb/src/planner/binder/expression/bind_star_expression.cpp +97 -19
  1292. package/src/duckdb/src/planner/binder/expression/bind_subquery_expression.cpp +74 -16
  1293. package/src/duckdb/src/planner/binder/expression/bind_unnest_expression.cpp +6 -6
  1294. package/src/duckdb/src/planner/binder/expression/bind_window_expression.cpp +49 -15
  1295. package/src/duckdb/src/planner/binder/query_node/bind_select_node.cpp +32 -23
  1296. package/src/duckdb/src/planner/binder/query_node/bind_setop_node.cpp +20 -3
  1297. package/src/duckdb/src/planner/binder/query_node/bind_table_macro_node.cpp +2 -2
  1298. package/src/duckdb/src/planner/binder/query_node/plan_query_node.cpp +3 -0
  1299. package/src/duckdb/src/planner/binder/query_node/plan_setop.cpp +6 -5
  1300. package/src/duckdb/src/planner/binder/query_node/plan_subquery.cpp +38 -19
  1301. package/src/duckdb/src/planner/binder/statement/bind_copy.cpp +2 -12
  1302. package/src/duckdb/src/planner/binder/statement/bind_create.cpp +117 -412
  1303. package/src/duckdb/src/planner/binder/statement/bind_create_table.cpp +423 -144
  1304. package/src/duckdb/src/planner/binder/statement/bind_delete.cpp +1 -1
  1305. package/src/duckdb/src/planner/binder/statement/bind_drop.cpp +5 -0
  1306. package/src/duckdb/src/planner/binder/statement/bind_execute.cpp +1 -1
  1307. package/src/duckdb/src/planner/binder/statement/bind_export.cpp +0 -4
  1308. package/src/duckdb/src/planner/binder/statement/bind_insert.cpp +31 -13
  1309. package/src/duckdb/src/planner/binder/statement/bind_pragma.cpp +1 -1
  1310. package/src/duckdb/src/planner/binder/statement/bind_simple.cpp +96 -27
  1311. package/src/duckdb/src/planner/binder/statement/bind_summarize.cpp +1 -1
  1312. package/src/duckdb/src/planner/binder/statement/bind_update.cpp +5 -3
  1313. package/src/duckdb/src/planner/binder/statement/bind_vacuum.cpp +7 -6
  1314. package/src/duckdb/src/planner/binder/tableref/bind_basetableref.cpp +36 -9
  1315. package/src/duckdb/src/planner/binder/tableref/bind_joinref.cpp +34 -34
  1316. package/src/duckdb/src/planner/binder/tableref/bind_pivot.cpp +72 -35
  1317. package/src/duckdb/src/planner/binder/tableref/bind_showref.cpp +99 -18
  1318. package/src/duckdb/src/planner/binder/tableref/bind_table_function.cpp +23 -11
  1319. package/src/duckdb/src/planner/binder/tableref/plan_joinref.cpp +22 -19
  1320. package/src/duckdb/src/planner/binder.cpp +23 -45
  1321. package/src/duckdb/src/planner/binding_alias.cpp +69 -0
  1322. package/src/duckdb/src/planner/bound_parameter_map.cpp +1 -1
  1323. package/src/duckdb/src/planner/bound_result_modifier.cpp +6 -2
  1324. package/src/duckdb/src/planner/collation_binding.cpp +38 -4
  1325. package/src/duckdb/src/planner/expression/bound_cast_expression.cpp +17 -5
  1326. package/src/duckdb/src/planner/expression/bound_expression.cpp +1 -1
  1327. package/src/duckdb/src/planner/expression/bound_function_expression.cpp +8 -1
  1328. package/src/duckdb/src/planner/expression/bound_parameter_expression.cpp +2 -2
  1329. package/src/duckdb/src/planner/expression/bound_window_expression.cpp +24 -4
  1330. package/src/duckdb/src/planner/expression.cpp +7 -1
  1331. package/src/duckdb/src/planner/expression_binder/aggregate_binder.cpp +1 -1
  1332. package/src/duckdb/src/planner/expression_binder/base_select_binder.cpp +2 -2
  1333. package/src/duckdb/src/planner/expression_binder/group_binder.cpp +2 -2
  1334. package/src/duckdb/src/planner/expression_binder/having_binder.cpp +16 -0
  1335. package/src/duckdb/src/planner/expression_binder/index_binder.cpp +53 -1
  1336. package/src/duckdb/src/planner/expression_binder/lateral_binder.cpp +3 -3
  1337. package/src/duckdb/src/planner/expression_binder/order_binder.cpp +8 -8
  1338. package/src/duckdb/src/planner/expression_binder/relation_binder.cpp +1 -1
  1339. package/src/duckdb/src/planner/expression_binder/select_bind_state.cpp +2 -2
  1340. package/src/duckdb/src/planner/expression_binder/table_function_binder.cpp +1 -1
  1341. package/src/duckdb/src/planner/expression_binder/update_binder.cpp +1 -1
  1342. package/src/duckdb/src/planner/expression_binder.cpp +7 -7
  1343. package/src/duckdb/src/planner/expression_iterator.cpp +6 -3
  1344. package/src/duckdb/src/planner/filter/constant_filter.cpp +17 -2
  1345. package/src/duckdb/src/planner/filter/dynamic_filter.cpp +68 -0
  1346. package/src/duckdb/src/planner/filter/in_filter.cpp +84 -0
  1347. package/src/duckdb/src/planner/filter/null_filter.cpp +1 -2
  1348. package/src/duckdb/src/planner/filter/optional_filter.cpp +29 -0
  1349. package/src/duckdb/src/planner/filter/struct_filter.cpp +11 -6
  1350. package/src/duckdb/src/planner/joinside.cpp +6 -5
  1351. package/src/duckdb/src/planner/logical_operator.cpp +4 -1
  1352. package/src/duckdb/src/planner/logical_operator_visitor.cpp +68 -2
  1353. package/src/duckdb/src/planner/operator/logical_comparison_join.cpp +23 -0
  1354. package/src/duckdb/src/planner/operator/logical_create_index.cpp +16 -12
  1355. package/src/duckdb/src/planner/operator/logical_filter.cpp +1 -1
  1356. package/src/duckdb/src/planner/operator/logical_get.cpp +48 -25
  1357. package/src/duckdb/src/planner/operator/logical_insert.cpp +1 -1
  1358. package/src/duckdb/src/planner/operator/logical_join.cpp +1 -1
  1359. package/src/duckdb/src/planner/operator/logical_order.cpp +4 -11
  1360. package/src/duckdb/src/planner/operator/logical_top_n.cpp +7 -0
  1361. package/src/duckdb/src/planner/subquery/flatten_dependent_join.cpp +33 -5
  1362. package/src/duckdb/src/planner/subquery/rewrite_correlated_expressions.cpp +2 -2
  1363. package/src/duckdb/src/planner/table_binding.cpp +74 -36
  1364. package/src/duckdb/src/planner/table_filter.cpp +5 -8
  1365. package/src/duckdb/src/storage/arena_allocator.cpp +5 -4
  1366. package/src/duckdb/src/storage/buffer/block_handle.cpp +88 -17
  1367. package/src/duckdb/src/storage/buffer/block_manager.cpp +34 -26
  1368. package/src/duckdb/src/storage/buffer/buffer_handle.cpp +2 -2
  1369. package/src/duckdb/src/storage/buffer/buffer_pool.cpp +70 -49
  1370. package/src/duckdb/src/storage/buffer_manager.cpp +4 -0
  1371. package/src/duckdb/src/storage/checkpoint_manager.cpp +24 -5
  1372. package/src/duckdb/src/storage/compression/bitpacking.cpp +14 -16
  1373. package/src/duckdb/src/storage/compression/dictionary/analyze.cpp +54 -0
  1374. package/src/duckdb/src/storage/compression/dictionary/common.cpp +90 -0
  1375. package/src/duckdb/src/storage/compression/dictionary/compression.cpp +174 -0
  1376. package/src/duckdb/src/storage/compression/dictionary/decompression.cpp +115 -0
  1377. package/src/duckdb/src/storage/compression/dictionary_compression.cpp +53 -545
  1378. package/src/duckdb/src/storage/compression/empty_validity.cpp +15 -0
  1379. package/src/duckdb/src/storage/compression/fixed_size_uncompressed.cpp +25 -16
  1380. package/src/duckdb/src/storage/compression/fsst.cpp +101 -47
  1381. package/src/duckdb/src/storage/compression/numeric_constant.cpp +92 -2
  1382. package/src/duckdb/src/storage/compression/rle.cpp +216 -46
  1383. package/src/duckdb/src/storage/compression/roaring/analyze.cpp +179 -0
  1384. package/src/duckdb/src/storage/compression/roaring/common.cpp +282 -0
  1385. package/src/duckdb/src/storage/compression/roaring/compress.cpp +481 -0
  1386. package/src/duckdb/src/storage/compression/roaring/metadata.cpp +262 -0
  1387. package/src/duckdb/src/storage/compression/roaring/scan.cpp +364 -0
  1388. package/src/duckdb/src/storage/compression/string_uncompressed.cpp +47 -65
  1389. package/src/duckdb/src/storage/compression/validity_uncompressed.cpp +102 -39
  1390. package/src/duckdb/src/storage/compression/zstd.cpp +1049 -0
  1391. package/src/duckdb/src/storage/data_table.cpp +312 -172
  1392. package/src/duckdb/src/storage/local_storage.cpp +104 -46
  1393. package/src/duckdb/src/storage/metadata/metadata_manager.cpp +1 -1
  1394. package/src/duckdb/src/storage/serialization/serialize_logical_operator.cpp +7 -3
  1395. package/src/duckdb/src/storage/serialization/serialize_nodes.cpp +138 -58
  1396. package/src/duckdb/src/storage/serialization/serialize_parse_info.cpp +14 -0
  1397. package/src/duckdb/src/storage/serialization/serialize_parsed_expression.cpp +19 -8
  1398. package/src/duckdb/src/storage/serialization/serialize_statement.cpp +2 -0
  1399. package/src/duckdb/src/storage/serialization/serialize_table_filter.cpp +43 -0
  1400. package/src/duckdb/src/storage/serialization/serialize_types.cpp +32 -5
  1401. package/src/duckdb/src/storage/single_file_block_manager.cpp +6 -8
  1402. package/src/duckdb/src/storage/standard_buffer_manager.cpp +82 -71
  1403. package/src/duckdb/src/storage/statistics/column_statistics.cpp +3 -3
  1404. package/src/duckdb/src/storage/statistics/distinct_statistics.cpp +18 -17
  1405. package/src/duckdb/src/storage/statistics/numeric_stats.cpp +34 -22
  1406. package/src/duckdb/src/storage/statistics/string_stats.cpp +14 -3
  1407. package/src/duckdb/src/storage/storage_info.cpp +72 -10
  1408. package/src/duckdb/src/storage/storage_manager.cpp +41 -47
  1409. package/src/duckdb/src/storage/table/array_column_data.cpp +7 -1
  1410. package/src/duckdb/src/storage/table/column_checkpoint_state.cpp +10 -9
  1411. package/src/duckdb/src/storage/table/column_data.cpp +105 -43
  1412. package/src/duckdb/src/storage/table/column_data_checkpointer.cpp +307 -132
  1413. package/src/duckdb/src/storage/table/column_segment.cpp +36 -13
  1414. package/src/duckdb/src/storage/table/list_column_data.cpp +4 -4
  1415. package/src/duckdb/src/storage/table/row_group.cpp +159 -66
  1416. package/src/duckdb/src/storage/table/row_group_collection.cpp +157 -68
  1417. package/src/duckdb/src/storage/table/row_version_manager.cpp +33 -10
  1418. package/src/duckdb/src/storage/table/scan_state.cpp +21 -7
  1419. package/src/duckdb/src/storage/table/standard_column_data.cpp +68 -5
  1420. package/src/duckdb/src/storage/table/struct_column_data.cpp +42 -4
  1421. package/src/duckdb/src/storage/table/table_statistics.cpp +91 -5
  1422. package/src/duckdb/src/storage/table/update_segment.cpp +287 -210
  1423. package/src/duckdb/src/storage/table_index_list.cpp +55 -58
  1424. package/src/duckdb/src/storage/temporary_file_manager.cpp +412 -149
  1425. package/src/duckdb/src/storage/wal_replay.cpp +132 -48
  1426. package/src/duckdb/src/storage/write_ahead_log.cpp +75 -48
  1427. package/src/duckdb/src/transaction/cleanup_state.cpp +0 -1
  1428. package/src/duckdb/src/transaction/commit_state.cpp +23 -14
  1429. package/src/duckdb/src/transaction/duck_transaction.cpp +29 -25
  1430. package/src/duckdb/src/transaction/duck_transaction_manager.cpp +18 -6
  1431. package/src/duckdb/src/transaction/meta_transaction.cpp +3 -2
  1432. package/src/duckdb/src/transaction/rollback_state.cpp +5 -2
  1433. package/src/duckdb/src/transaction/transaction_context.cpp +9 -1
  1434. package/src/duckdb/src/transaction/undo_buffer.cpp +35 -27
  1435. package/src/duckdb/src/transaction/undo_buffer_allocator.cpp +72 -0
  1436. package/src/duckdb/src/transaction/wal_write_state.cpp +12 -10
  1437. package/src/duckdb/src/verification/copied_statement_verifier.cpp +7 -4
  1438. package/src/duckdb/src/verification/deserialized_statement_verifier.cpp +7 -5
  1439. package/src/duckdb/src/verification/external_statement_verifier.cpp +7 -4
  1440. package/src/duckdb/src/verification/fetch_row_verifier.cpp +7 -4
  1441. package/src/duckdb/src/verification/no_operator_caching_verifier.cpp +8 -4
  1442. package/src/duckdb/src/verification/parsed_statement_verifier.cpp +7 -4
  1443. package/src/duckdb/src/verification/prepared_statement_verifier.cpp +16 -12
  1444. package/src/duckdb/src/verification/statement_verifier.cpp +20 -15
  1445. package/src/duckdb/src/verification/unoptimized_statement_verifier.cpp +7 -4
  1446. package/src/duckdb/third_party/fsst/libfsst.hpp +1 -0
  1447. package/src/duckdb/third_party/httplib/httplib.hpp +15 -22
  1448. package/src/duckdb/third_party/libpg_query/include/nodes/parsenodes.hpp +4 -2
  1449. package/src/duckdb/third_party/libpg_query/pg_functions.cpp +2 -4
  1450. package/src/duckdb/third_party/libpg_query/src_backend_parser_gram.cpp +14278 -13832
  1451. package/src/duckdb/third_party/parquet/parquet_types.cpp +3410 -1686
  1452. package/src/duckdb/third_party/parquet/parquet_types.h +1585 -1204
  1453. package/src/duckdb/third_party/skiplist/SkipList.h +0 -1
  1454. package/src/duckdb/third_party/snappy/snappy-stubs-internal.h +13 -15
  1455. package/src/duckdb/third_party/zstd/common/debug.cpp +36 -0
  1456. package/src/duckdb/third_party/zstd/common/entropy_common.cpp +173 -49
  1457. package/src/duckdb/third_party/zstd/common/error_private.cpp +11 -3
  1458. package/src/duckdb/third_party/zstd/common/fse_decompress.cpp +126 -97
  1459. package/src/duckdb/third_party/zstd/common/pool.cpp +376 -0
  1460. package/src/duckdb/third_party/zstd/common/threading.cpp +193 -0
  1461. package/src/duckdb/third_party/zstd/common/xxhash.cpp +18 -14
  1462. package/src/duckdb/third_party/zstd/common/zstd_common.cpp +3 -38
  1463. package/src/duckdb/third_party/zstd/compress/fse_compress.cpp +93 -165
  1464. package/src/duckdb/third_party/zstd/compress/hist.cpp +28 -31
  1465. package/src/duckdb/third_party/zstd/compress/huf_compress.cpp +957 -291
  1466. package/src/duckdb/third_party/zstd/compress/zstd_compress.cpp +3988 -1124
  1467. package/src/duckdb/third_party/zstd/compress/zstd_compress_literals.cpp +120 -43
  1468. package/src/duckdb/third_party/zstd/compress/zstd_compress_sequences.cpp +47 -23
  1469. package/src/duckdb/third_party/zstd/compress/zstd_compress_superblock.cpp +274 -424
  1470. package/src/duckdb/third_party/zstd/compress/zstd_double_fast.cpp +403 -153
  1471. package/src/duckdb/third_party/zstd/compress/zstd_fast.cpp +741 -268
  1472. package/src/duckdb/third_party/zstd/compress/zstd_lazy.cpp +1339 -278
  1473. package/src/duckdb/third_party/zstd/compress/zstd_ldm.cpp +334 -222
  1474. package/src/duckdb/third_party/zstd/compress/zstd_opt.cpp +674 -298
  1475. package/src/duckdb/third_party/zstd/compress/zstdmt_compress.cpp +1885 -0
  1476. package/src/duckdb/third_party/zstd/decompress/huf_decompress.cpp +1247 -586
  1477. package/src/duckdb/third_party/zstd/decompress/zstd_ddict.cpp +18 -17
  1478. package/src/duckdb/third_party/zstd/decompress/zstd_decompress.cpp +724 -270
  1479. package/src/duckdb/third_party/zstd/decompress/zstd_decompress_block.cpp +1193 -393
  1480. package/src/duckdb/third_party/zstd/deprecated/zbuff_common.cpp +30 -0
  1481. package/src/duckdb/third_party/zstd/deprecated/zbuff_compress.cpp +171 -0
  1482. package/src/duckdb/third_party/zstd/deprecated/zbuff_decompress.cpp +80 -0
  1483. package/src/duckdb/third_party/zstd/dict/cover.cpp +1271 -0
  1484. package/src/duckdb/third_party/zstd/dict/divsufsort.cpp +1916 -0
  1485. package/src/duckdb/third_party/zstd/dict/fastcover.cpp +775 -0
  1486. package/src/duckdb/third_party/zstd/dict/zdict.cpp +1139 -0
  1487. package/src/duckdb/third_party/zstd/include/zdict.h +473 -0
  1488. package/src/duckdb/third_party/zstd/include/zstd/common/allocations.h +58 -0
  1489. package/src/duckdb/third_party/zstd/include/zstd/common/bits.h +204 -0
  1490. package/src/duckdb/third_party/zstd/include/zstd/common/bitstream.h +88 -85
  1491. package/src/duckdb/third_party/zstd/include/zstd/common/compiler.h +243 -47
  1492. package/src/duckdb/third_party/zstd/include/zstd/common/cpu.h +253 -0
  1493. package/src/duckdb/third_party/zstd/include/zstd/common/debug.h +31 -31
  1494. package/src/duckdb/third_party/zstd/include/zstd/common/error_private.h +94 -6
  1495. package/src/duckdb/third_party/zstd/include/zstd/common/fse.h +424 -64
  1496. package/src/duckdb/third_party/zstd/include/zstd/common/huf.h +255 -70
  1497. package/src/duckdb/third_party/zstd/include/zstd/common/mem.h +125 -85
  1498. package/src/duckdb/third_party/zstd/include/zstd/common/pool.h +84 -0
  1499. package/src/duckdb/third_party/zstd/include/zstd/common/portability_macros.h +158 -0
  1500. package/src/duckdb/third_party/zstd/include/zstd/common/threading.h +152 -0
  1501. package/src/duckdb/third_party/zstd/include/zstd/common/{xxhash.h → xxhash.hpp} +0 -1
  1502. package/src/duckdb/third_party/zstd/include/zstd/common/{xxhash_static.h → xxhash_static.hpp} +1 -1
  1503. package/src/duckdb/third_party/zstd/include/zstd/common/zstd_deps.h +122 -0
  1504. package/src/duckdb/third_party/zstd/include/zstd/common/zstd_internal.h +143 -174
  1505. package/src/duckdb/third_party/zstd/include/zstd/common/zstd_trace.h +159 -0
  1506. package/src/duckdb/third_party/zstd/include/zstd/compress/clevels.h +136 -0
  1507. package/src/duckdb/third_party/zstd/include/zstd/compress/hist.h +4 -4
  1508. package/src/duckdb/third_party/zstd/include/zstd/compress/zstd_compress_internal.h +631 -220
  1509. package/src/duckdb/third_party/zstd/include/zstd/compress/zstd_compress_literals.h +17 -7
  1510. package/src/duckdb/third_party/zstd/include/zstd/compress/zstd_compress_sequences.h +2 -2
  1511. package/src/duckdb/third_party/zstd/include/zstd/compress/zstd_compress_superblock.h +3 -2
  1512. package/src/duckdb/third_party/zstd/include/zstd/compress/zstd_cwksp.h +256 -153
  1513. package/src/duckdb/third_party/zstd/include/zstd/compress/zstd_double_fast.h +16 -3
  1514. package/src/duckdb/third_party/zstd/include/zstd/compress/zstd_fast.h +4 -3
  1515. package/src/duckdb/third_party/zstd/include/zstd/compress/zstd_lazy.h +145 -11
  1516. package/src/duckdb/third_party/zstd/include/zstd/compress/zstd_ldm.h +14 -6
  1517. package/src/duckdb/third_party/zstd/include/zstd/compress/zstd_ldm_geartab.h +110 -0
  1518. package/src/duckdb/third_party/zstd/include/zstd/compress/zstd_opt.h +33 -9
  1519. package/src/duckdb/third_party/zstd/include/zstd/compress/zstdmt_compress.h +107 -0
  1520. package/src/duckdb/third_party/zstd/include/zstd/decompress/zstd_ddict.h +4 -3
  1521. package/src/duckdb/third_party/zstd/include/zstd/decompress/zstd_decompress_block.h +20 -6
  1522. package/src/duckdb/third_party/zstd/include/zstd/decompress/zstd_decompress_internal.h +88 -16
  1523. package/src/duckdb/third_party/zstd/include/zstd/deprecated/zbuff.h +214 -0
  1524. package/src/duckdb/third_party/zstd/include/zstd/dict/cover.h +156 -0
  1525. package/src/duckdb/third_party/zstd/include/zstd/dict/divsufsort.h +62 -0
  1526. package/src/duckdb/third_party/zstd/include/zstd.h +2171 -93
  1527. package/src/duckdb/third_party/zstd/include/{zstd/common/zstd_errors.h → zstd_errors.h} +32 -10
  1528. package/src/duckdb/ub_extension_core_functions_aggregate_algebraic.cpp +8 -0
  1529. package/src/duckdb/ub_extension_core_functions_aggregate_distributive.cpp +20 -0
  1530. package/src/duckdb/ub_extension_core_functions_aggregate_holistic.cpp +12 -0
  1531. package/src/duckdb/ub_extension_core_functions_aggregate_nested.cpp +6 -0
  1532. package/src/duckdb/ub_extension_core_functions_aggregate_regression.cpp +14 -0
  1533. package/src/duckdb/ub_extension_core_functions_scalar_array.cpp +4 -0
  1534. package/src/duckdb/ub_extension_core_functions_scalar_bit.cpp +2 -0
  1535. package/src/duckdb/ub_extension_core_functions_scalar_blob.cpp +4 -0
  1536. package/src/duckdb/ub_extension_core_functions_scalar_date.cpp +20 -0
  1537. package/src/duckdb/ub_extension_core_functions_scalar_debug.cpp +2 -0
  1538. package/src/duckdb/ub_extension_core_functions_scalar_enum.cpp +2 -0
  1539. package/src/duckdb/ub_extension_core_functions_scalar_generic.cpp +18 -0
  1540. package/src/duckdb/ub_extension_core_functions_scalar_list.cpp +22 -0
  1541. package/src/duckdb/ub_extension_core_functions_scalar_map.cpp +14 -0
  1542. package/src/duckdb/ub_extension_core_functions_scalar_math.cpp +2 -0
  1543. package/src/duckdb/ub_extension_core_functions_scalar_operators.cpp +2 -0
  1544. package/src/duckdb/ub_extension_core_functions_scalar_random.cpp +4 -0
  1545. package/src/duckdb/ub_extension_core_functions_scalar_string.cpp +48 -0
  1546. package/src/duckdb/ub_extension_core_functions_scalar_struct.cpp +2 -0
  1547. package/src/duckdb/ub_extension_core_functions_scalar_union.cpp +6 -0
  1548. package/src/duckdb/ub_src_common.cpp +4 -0
  1549. package/src/duckdb/ub_src_common_arrow.cpp +3 -1
  1550. package/src/duckdb/ub_src_execution.cpp +0 -6
  1551. package/src/duckdb/ub_src_execution_operator_aggregate.cpp +2 -0
  1552. package/src/duckdb/ub_src_execution_operator_csv_scanner_encode.cpp +2 -0
  1553. package/src/duckdb/ub_src_execution_operator_csv_scanner_util.cpp +2 -0
  1554. package/src/duckdb/ub_src_execution_sample.cpp +4 -0
  1555. package/src/duckdb/ub_src_function.cpp +6 -0
  1556. package/src/duckdb/ub_src_function_aggregate.cpp +0 -2
  1557. package/src/duckdb/ub_src_function_aggregate_distributive.cpp +3 -1
  1558. package/src/duckdb/ub_src_function_scalar.cpp +2 -8
  1559. package/src/duckdb/ub_src_function_scalar_date.cpp +2 -0
  1560. package/src/duckdb/ub_src_function_scalar_generic.cpp +2 -2
  1561. package/src/duckdb/ub_src_function_scalar_map.cpp +2 -0
  1562. package/src/duckdb/ub_src_function_scalar_operator.cpp +8 -0
  1563. package/src/duckdb/ub_src_function_scalar_string.cpp +10 -0
  1564. package/src/duckdb/ub_src_function_scalar_struct.cpp +4 -0
  1565. package/src/duckdb/ub_src_function_scalar_system.cpp +2 -0
  1566. package/src/duckdb/ub_src_function_table_system.cpp +6 -0
  1567. package/src/duckdb/ub_src_function_window.cpp +36 -0
  1568. package/src/duckdb/ub_src_logging.cpp +8 -0
  1569. package/src/duckdb/ub_src_main_settings.cpp +3 -1
  1570. package/src/duckdb/ub_src_optimizer.cpp +8 -0
  1571. package/src/duckdb/ub_src_optimizer_pushdown.cpp +2 -0
  1572. package/src/duckdb/ub_src_optimizer_rule.cpp +2 -0
  1573. package/src/duckdb/ub_src_parser.cpp +2 -0
  1574. package/src/duckdb/ub_src_parser_parsed_data.cpp +2 -0
  1575. package/src/duckdb/ub_src_planner.cpp +2 -0
  1576. package/src/duckdb/ub_src_planner_filter.cpp +6 -0
  1577. package/src/duckdb/ub_src_storage_compression.cpp +4 -0
  1578. package/src/duckdb/ub_src_storage_compression_dictionary.cpp +8 -0
  1579. package/src/duckdb/ub_src_storage_compression_roaring.cpp +10 -0
  1580. package/src/duckdb/ub_src_transaction.cpp +2 -0
  1581. package/vendor.py +1 -1
  1582. package/src/duckdb/extension/json/yyjson/include/yyjson.hpp +0 -6003
  1583. package/src/duckdb/extension/json/yyjson/yyjson.cpp +0 -8218
  1584. package/src/duckdb/src/common/arrow/appender/list_data.cpp +0 -78
  1585. package/src/duckdb/src/common/arrow/appender/map_data.cpp +0 -91
  1586. package/src/duckdb/src/common/cycle_counter.cpp +0 -76
  1587. package/src/duckdb/src/common/field_writer.cpp +0 -97
  1588. package/src/duckdb/src/common/http_state.cpp +0 -95
  1589. package/src/duckdb/src/common/preserved_error.cpp +0 -87
  1590. package/src/duckdb/src/common/row_operations/row_match.cpp +0 -359
  1591. package/src/duckdb/src/common/serializer/buffered_deserializer.cpp +0 -27
  1592. package/src/duckdb/src/common/serializer/buffered_serializer.cpp +0 -36
  1593. package/src/duckdb/src/common/serializer/format_serializer.cpp +0 -15
  1594. package/src/duckdb/src/common/serializer.cpp +0 -24
  1595. package/src/duckdb/src/common/types/chunk_collection.cpp +0 -190
  1596. package/src/duckdb/src/core_functions/aggregate/distributive/entropy.cpp +0 -183
  1597. package/src/duckdb/src/core_functions/scalar/date/current.cpp +0 -54
  1598. package/src/duckdb/src/core_functions/scalar/list/list_cosine_similarity.cpp +0 -78
  1599. package/src/duckdb/src/core_functions/scalar/list/list_inner_product.cpp +0 -70
  1600. package/src/duckdb/src/core_functions/scalar/list/list_lambdas.cpp +0 -412
  1601. package/src/duckdb/src/core_functions/scalar/secret/which_secret.cpp +0 -28
  1602. package/src/duckdb/src/core_functions/scalar/string/jaro_winkler.cpp +0 -71
  1603. package/src/duckdb/src/execution/index/art/fixed_size_allocator.cpp +0 -238
  1604. package/src/duckdb/src/execution/index/art/node16.cpp +0 -196
  1605. package/src/duckdb/src/execution/index/art/node4.cpp +0 -189
  1606. package/src/duckdb/src/execution/index/unknown_index.cpp +0 -65
  1607. package/src/duckdb/src/execution/operator/csv_scanner/base_csv_reader.cpp +0 -595
  1608. package/src/duckdb/src/execution/operator/csv_scanner/buffered_csv_reader.cpp +0 -434
  1609. package/src/duckdb/src/execution/operator/csv_scanner/csv_buffer.cpp +0 -89
  1610. package/src/duckdb/src/execution/operator/csv_scanner/csv_buffer_manager.cpp +0 -90
  1611. package/src/duckdb/src/execution/operator/csv_scanner/csv_file_handle.cpp +0 -95
  1612. package/src/duckdb/src/execution/operator/csv_scanner/csv_reader_options.cpp +0 -494
  1613. package/src/duckdb/src/execution/operator/csv_scanner/csv_state_machine.cpp +0 -35
  1614. package/src/duckdb/src/execution/operator/csv_scanner/csv_state_machine_cache.cpp +0 -99
  1615. package/src/duckdb/src/execution/operator/csv_scanner/parallel_csv_reader.cpp +0 -689
  1616. package/src/duckdb/src/execution/operator/join/physical_index_join.cpp +0 -242
  1617. package/src/duckdb/src/execution/operator/persistent/base_csv_reader.cpp +0 -695
  1618. package/src/duckdb/src/execution/operator/persistent/buffered_csv_reader.cpp +0 -1487
  1619. package/src/duckdb/src/execution/operator/persistent/csv_buffer.cpp +0 -72
  1620. package/src/duckdb/src/execution/operator/persistent/csv_file_handle.cpp +0 -158
  1621. package/src/duckdb/src/execution/operator/persistent/csv_reader_options.cpp +0 -280
  1622. package/src/duckdb/src/execution/operator/persistent/parallel_csv_reader.cpp +0 -666
  1623. package/src/duckdb/src/execution/operator/persistent/physical_fixed_batch_copy.cpp +0 -499
  1624. package/src/duckdb/src/execution/operator/schema/physical_create_index.cpp +0 -207
  1625. package/src/duckdb/src/execution/partitionable_hashtable.cpp +0 -207
  1626. package/src/duckdb/src/execution/physical_plan/plan_limit_percent.cpp +0 -18
  1627. package/src/duckdb/src/execution/physical_plan/plan_show_select.cpp +0 -47
  1628. package/src/duckdb/src/execution/reservoir_sample.cpp +0 -324
  1629. package/src/duckdb/src/execution/window_executor.cpp +0 -1830
  1630. package/src/duckdb/src/execution/window_segment_tree.cpp +0 -2073
  1631. package/src/duckdb/src/extension_forward_decl/icu.cpp +0 -59
  1632. package/src/duckdb/src/function/aggregate/distributive_functions.cpp +0 -15
  1633. package/src/duckdb/src/function/scalar/compressed_materialization_functions.cpp +0 -29
  1634. package/src/duckdb/src/function/scalar/generic_functions.cpp +0 -11
  1635. package/src/duckdb/src/function/scalar/list/list_concat.cpp +0 -143
  1636. package/src/duckdb/src/function/scalar/operators.cpp +0 -14
  1637. package/src/duckdb/src/function/scalar/sequence_functions.cpp +0 -10
  1638. package/src/duckdb/src/function/scalar/string_functions.cpp +0 -22
  1639. package/src/duckdb/src/function/table/pragma_detailed_profiling_output.cpp +0 -173
  1640. package/src/duckdb/src/function/table/pragma_last_profiling_output.cpp +0 -101
  1641. package/src/duckdb/src/include/duckdb/catalog/mapping_value.hpp +0 -92
  1642. package/src/duckdb/src/include/duckdb/common/arrow/arrow_types_extension.hpp +0 -42
  1643. package/src/duckdb/src/include/duckdb/common/cycle_counter.hpp +0 -68
  1644. package/src/duckdb/src/include/duckdb/common/enums/index_type.hpp +0 -34
  1645. package/src/duckdb/src/include/duckdb/common/http_state.hpp +0 -113
  1646. package/src/duckdb/src/include/duckdb/common/platform.h +0 -58
  1647. package/src/duckdb/src/include/duckdb/common/preserved_error.hpp +0 -59
  1648. package/src/duckdb/src/include/duckdb/common/serializer/deserialization_data.hpp +0 -192
  1649. package/src/duckdb/src/include/duckdb/common/types/chunk_collection.hpp +0 -137
  1650. package/src/duckdb/src/include/duckdb/execution/index/art/node16.hpp +0 -65
  1651. package/src/duckdb/src/include/duckdb/execution/index/art/node4.hpp +0 -63
  1652. package/src/duckdb/src/include/duckdb/execution/index/unknown_index.hpp +0 -65
  1653. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/buffer_manager/csv_buffer.hpp +0 -103
  1654. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/buffer_manager/csv_buffer_manager.hpp +0 -74
  1655. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/buffer_manager/csv_file_handle.hpp +0 -60
  1656. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/csv_sniffer.hpp +0 -253
  1657. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/options/csv_option.hpp +0 -155
  1658. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/options/csv_reader_options.hpp +0 -163
  1659. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/options/state_machine_options.hpp +0 -35
  1660. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/scanner/base_scanner.hpp +0 -228
  1661. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/scanner/column_count_scanner.hpp +0 -70
  1662. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/scanner/scanner_boundary.hpp +0 -93
  1663. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/scanner/skip_scanner.hpp +0 -60
  1664. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/scanner/string_value_scanner.hpp +0 -197
  1665. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/sniffer/quote_rules.hpp +0 -21
  1666. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/state_machine/csv_state.hpp +0 -30
  1667. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/state_machine/csv_state_machine.hpp +0 -99
  1668. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/state_machine/csv_state_machine_cache.hpp +0 -87
  1669. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/table_function/csv_file_scanner.hpp +0 -70
  1670. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/table_function/global_csv_state.hpp +0 -80
  1671. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/util/csv_casting.hpp +0 -137
  1672. package/src/duckdb/src/include/duckdb/execution/operator/csv_scanner/util/csv_error.hpp +0 -104
  1673. package/src/duckdb/src/include/duckdb/execution/operator/join/physical_index_join.hpp +0 -79
  1674. package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/base_csv_reader.hpp +0 -119
  1675. package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/buffered_csv_reader.hpp +0 -72
  1676. package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/csv_buffer.hpp +0 -110
  1677. package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/csv_buffer_manager.hpp +0 -103
  1678. package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/csv_file_handle.hpp +0 -59
  1679. package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/csv_line_info.hpp +0 -46
  1680. package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/csv_reader_options.hpp +0 -210
  1681. package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/csv_sniffer.hpp +0 -131
  1682. package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/csv_state.hpp +0 -28
  1683. package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/csv_state_machine.hpp +0 -70
  1684. package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/csv_state_machine_cache.hpp +0 -65
  1685. package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/parallel_csv_reader.hpp +0 -167
  1686. package/src/duckdb/src/include/duckdb/execution/operator/scan/csv/quote_rules.hpp +0 -21
  1687. package/src/duckdb/src/include/duckdb/execution/window_executor.hpp +0 -343
  1688. package/src/duckdb/src/include/duckdb/execution/window_segment_tree.hpp +0 -165
  1689. package/src/duckdb/src/include/duckdb/optimizer/column_lifetime_optimizer.hpp +0 -45
  1690. package/src/duckdb/src/include/duckdb/optimizer/join_order/estimated_properties.hpp +0 -57
  1691. package/src/duckdb/src/include/duckdb/parser/parsed_data/comment_on_info.hpp +0 -45
  1692. package/src/duckdb/src/include/duckdb/parser/statement/show_statement.hpp +0 -32
  1693. package/src/duckdb/src/include/duckdb/planner/operator/logical_limit_percent.hpp +0 -49
  1694. package/src/duckdb/src/include/duckdb/planner/operator/logical_show.hpp +0 -42
  1695. package/src/duckdb/src/main/settings/settings.cpp +0 -2056
  1696. package/src/duckdb/src/optimizer/join_order/estimated_properties.cpp +0 -36
  1697. package/src/duckdb/src/parser/parsed_data/comment_on_info.cpp +0 -19
  1698. package/src/duckdb/src/parser/statement/show_statement.cpp +0 -15
  1699. package/src/duckdb/src/planner/binder/statement/bind_show.cpp +0 -30
  1700. package/src/duckdb/src/planner/operator/logical_limit_percent.cpp +0 -14
  1701. package/src/duckdb/src/storage/statistics/numeric_stats_union.cpp +0 -70
  1702. package/src/duckdb/third_party/fsst/fsst_avx512.cpp +0 -140
  1703. package/src/duckdb/third_party/fsst/fsst_avx512.inc +0 -57
  1704. package/src/duckdb/third_party/fsst/fsst_avx512_unroll1.inc +0 -57
  1705. package/src/duckdb/third_party/fsst/fsst_avx512_unroll2.inc +0 -114
  1706. package/src/duckdb/third_party/fsst/fsst_avx512_unroll3.inc +0 -171
  1707. package/src/duckdb/third_party/fsst/fsst_avx512_unroll4.inc +0 -228
  1708. package/src/duckdb/third_party/parquet/parquet_constants.cpp +0 -17
  1709. package/src/duckdb/third_party/parquet/parquet_constants.h +0 -24
  1710. package/src/duckdb/third_party/re2/util/pod_array.h +0 -55
  1711. package/src/duckdb/third_party/re2/util/sparse_array.h +0 -392
  1712. package/src/duckdb/third_party/re2/util/sparse_set.h +0 -264
  1713. package/src/duckdb/third_party/zstd/include/zstd/common/fse_static.h +0 -421
  1714. package/src/duckdb/third_party/zstd/include/zstd/common/huf_static.h +0 -238
  1715. package/src/duckdb/third_party/zstd/include/zstd_static.h +0 -1070
  1716. package/src/duckdb/ub_src_core_functions.cpp +0 -6
  1717. package/src/duckdb/ub_src_core_functions_aggregate_algebraic.cpp +0 -8
  1718. package/src/duckdb/ub_src_core_functions_aggregate_distributive.cpp +0 -24
  1719. package/src/duckdb/ub_src_core_functions_aggregate_holistic.cpp +0 -12
  1720. package/src/duckdb/ub_src_core_functions_aggregate_nested.cpp +0 -6
  1721. package/src/duckdb/ub_src_core_functions_aggregate_regression.cpp +0 -14
  1722. package/src/duckdb/ub_src_core_functions_scalar_array.cpp +0 -4
  1723. package/src/duckdb/ub_src_core_functions_scalar_bit.cpp +0 -2
  1724. package/src/duckdb/ub_src_core_functions_scalar_blob.cpp +0 -6
  1725. package/src/duckdb/ub_src_core_functions_scalar_date.cpp +0 -22
  1726. package/src/duckdb/ub_src_core_functions_scalar_debug.cpp +0 -2
  1727. package/src/duckdb/ub_src_core_functions_scalar_enum.cpp +0 -2
  1728. package/src/duckdb/ub_src_core_functions_scalar_generic.cpp +0 -18
  1729. package/src/duckdb/ub_src_core_functions_scalar_list.cpp +0 -22
  1730. package/src/duckdb/ub_src_core_functions_scalar_map.cpp +0 -16
  1731. package/src/duckdb/ub_src_core_functions_scalar_math.cpp +0 -2
  1732. package/src/duckdb/ub_src_core_functions_scalar_operators.cpp +0 -2
  1733. package/src/duckdb/ub_src_core_functions_scalar_random.cpp +0 -4
  1734. package/src/duckdb/ub_src_core_functions_scalar_secret.cpp +0 -2
  1735. package/src/duckdb/ub_src_core_functions_scalar_string.cpp +0 -58
  1736. package/src/duckdb/ub_src_core_functions_scalar_struct.cpp +0 -4
  1737. package/src/duckdb/ub_src_core_functions_scalar_union.cpp +0 -6
  1738. package/src/duckdb/ub_src_execution_operator_csv_scanner.cpp +0 -18
  1739. package/src/duckdb/ub_src_function_scalar_operators.cpp +0 -8
  1740. /package/src/duckdb/{src/include/duckdb → extension/core_functions/include}/core_functions/aggregate/algebraic/covar.hpp +0 -0
  1741. /package/src/duckdb/{src/include/duckdb → extension/core_functions/include}/core_functions/aggregate/algebraic/stddev.hpp +0 -0
  1742. /package/src/duckdb/{src/include/duckdb → extension/core_functions/include}/core_functions/aggregate/sum_helpers.hpp +0 -0
  1743. /package/src/duckdb/{src/include/duckdb → extension/core_functions/include}/core_functions/array_kernels.hpp +0 -0
  1744. /package/src/duckdb/{src/include/duckdb → extension/core_functions/include}/core_functions/function_list.hpp +0 -0
  1745. /package/src/duckdb/{src/include/duckdb → extension/core_functions/include}/core_functions/scalar/secret_functions.hpp +0 -0
  1746. /package/src/duckdb/src/function/scalar/{operators → operator}/multiply.cpp +0 -0
  1747. /package/src/duckdb/src/function/scalar/{operators → operator}/subtract.cpp +0 -0
@@ -1,5 +1,5 @@
1
1
  /*
2
- * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
2
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
3
3
  * All rights reserved.
4
4
  *
5
5
  * This source code is licensed under both the BSD-style license (found in the
@@ -15,288 +15,11 @@
15
15
 
16
16
  #include "zstd/common/zstd_internal.h" /* ZSTD_getSequenceLength */
17
17
  #include "zstd/compress/hist.h" /* HIST_countFast_wksp */
18
- #include "zstd/compress/zstd_compress_internal.h"
18
+ #include "zstd/compress/zstd_compress_internal.h" /* ZSTD_[huf|fse|entropy]CTablesMetadata_t */
19
19
  #include "zstd/compress/zstd_compress_sequences.h"
20
20
  #include "zstd/compress/zstd_compress_literals.h"
21
21
 
22
22
  namespace duckdb_zstd {
23
- /*-*************************************
24
- * Superblock entropy buffer structs
25
- ***************************************/
26
- /** ZSTD_hufCTablesMetadata_t :
27
- * Stores Literals Block Type for a super-block in hType, and
28
- * huffman tree description in hufDesBuffer.
29
- * hufDesSize refers to the size of huffman tree description in bytes.
30
- * This metadata is populated in ZSTD_buildSuperBlockEntropy_literal() */
31
- typedef struct {
32
- symbolEncodingType_e hType;
33
- BYTE hufDesBuffer[500]; /* TODO give name to this value */
34
- size_t hufDesSize;
35
- } ZSTD_hufCTablesMetadata_t;
36
-
37
- /** ZSTD_fseCTablesMetadata_t :
38
- * Stores symbol compression modes for a super-block in {ll, ol, ml}Type, and
39
- * fse tables in fseTablesBuffer.
40
- * fseTablesSize refers to the size of fse tables in bytes.
41
- * This metadata is populated in ZSTD_buildSuperBlockEntropy_sequences() */
42
- typedef struct {
43
- symbolEncodingType_e llType;
44
- symbolEncodingType_e ofType;
45
- symbolEncodingType_e mlType;
46
- BYTE fseTablesBuffer[500]; /* TODO give name to this value */
47
- size_t fseTablesSize;
48
- size_t lastCountSize; /* This is to account for bug in 1.3.4. More detail in ZSTD_compressSubBlock_sequences() */
49
- } ZSTD_fseCTablesMetadata_t;
50
-
51
- typedef struct {
52
- ZSTD_hufCTablesMetadata_t hufMetadata;
53
- ZSTD_fseCTablesMetadata_t fseMetadata;
54
- } ZSTD_entropyCTablesMetadata_t;
55
-
56
-
57
- /** ZSTD_buildSuperBlockEntropy_literal() :
58
- * Builds entropy for the super-block literals.
59
- * Stores literals block type (raw, rle, compressed, repeat) and
60
- * huffman description table to hufMetadata.
61
- * @return : size of huffman description table or error code */
62
- static size_t ZSTD_buildSuperBlockEntropy_literal(void* const src, size_t srcSize,
63
- const ZSTD_hufCTables_t* prevHuf,
64
- ZSTD_hufCTables_t* nextHuf,
65
- ZSTD_hufCTablesMetadata_t* hufMetadata,
66
- const int disableLiteralsCompression,
67
- void* workspace, size_t wkspSize)
68
- {
69
- BYTE* const wkspStart = (BYTE*)workspace;
70
- BYTE* const wkspEnd = wkspStart + wkspSize;
71
- BYTE* const countWkspStart = wkspStart;
72
- unsigned* const countWksp = (unsigned*)workspace;
73
- const size_t countWkspSize = (HUF_SYMBOLVALUE_MAX + 1) * sizeof(unsigned);
74
- BYTE* const nodeWksp = countWkspStart + countWkspSize;
75
- const size_t nodeWkspSize = wkspEnd-nodeWksp;
76
- unsigned maxSymbolValue = 255;
77
- unsigned huffLog = HUF_TABLELOG_DEFAULT;
78
- HUF_repeat repeat = prevHuf->repeatMode;
79
-
80
- DEBUGLOG(5, "ZSTD_buildSuperBlockEntropy_literal (srcSize=%zu)", srcSize);
81
-
82
- /* Prepare nextEntropy assuming reusing the existing table */
83
- memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
84
-
85
- if (disableLiteralsCompression) {
86
- DEBUGLOG(5, "set_basic - disabled");
87
- hufMetadata->hType = set_basic;
88
- return 0;
89
- }
90
-
91
- /* small ? don't even attempt compression (speed opt) */
92
- # define COMPRESS_LITERALS_SIZE_MIN 63
93
- { size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN;
94
- if (srcSize <= minLitSize) {
95
- DEBUGLOG(5, "set_basic - too small");
96
- hufMetadata->hType = set_basic;
97
- return 0;
98
- }
99
- }
100
-
101
- /* Scan input and build symbol stats */
102
- { size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)src, srcSize, workspace, wkspSize);
103
- FORWARD_IF_ERROR(largest, "HIST_count_wksp failed");
104
- if (largest == srcSize) {
105
- DEBUGLOG(5, "set_rle");
106
- hufMetadata->hType = set_rle;
107
- return 0;
108
- }
109
- if (largest <= (srcSize >> 7)+4) {
110
- DEBUGLOG(5, "set_basic - no gain");
111
- hufMetadata->hType = set_basic;
112
- return 0;
113
- }
114
- }
115
-
116
- /* Validate the previous Huffman table */
117
- if (repeat == HUF_repeat_check && !HUF_validateCTable((HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue)) {
118
- repeat = HUF_repeat_none;
119
- }
120
-
121
- /* Build Huffman Tree */
122
- memset(nextHuf->CTable, 0, sizeof(nextHuf->CTable));
123
- huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
124
- { size_t const maxBits = HUF_buildCTable_wksp((HUF_CElt*)nextHuf->CTable, countWksp,
125
- maxSymbolValue, huffLog,
126
- nodeWksp, nodeWkspSize);
127
- FORWARD_IF_ERROR(maxBits, "HUF_buildCTable_wksp");
128
- huffLog = (U32)maxBits;
129
- { /* Build and write the CTable */
130
- size_t const newCSize = HUF_estimateCompressedSize(
131
- (HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue);
132
- size_t const hSize = HUF_writeCTable(
133
- hufMetadata->hufDesBuffer, sizeof(hufMetadata->hufDesBuffer),
134
- (HUF_CElt*)nextHuf->CTable, maxSymbolValue, huffLog);
135
- /* Check against repeating the previous CTable */
136
- if (repeat != HUF_repeat_none) {
137
- size_t const oldCSize = HUF_estimateCompressedSize(
138
- (HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue);
139
- if (oldCSize < srcSize && (oldCSize <= hSize + newCSize || hSize + 12 >= srcSize)) {
140
- DEBUGLOG(5, "set_repeat - smaller");
141
- memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
142
- hufMetadata->hType = set_repeat;
143
- return 0;
144
- }
145
- }
146
- if (newCSize + hSize >= srcSize) {
147
- DEBUGLOG(5, "set_basic - no gains");
148
- memcpy(nextHuf, prevHuf, sizeof(*prevHuf));
149
- hufMetadata->hType = set_basic;
150
- return 0;
151
- }
152
- DEBUGLOG(5, "set_compressed (hSize=%u)", (U32)hSize);
153
- hufMetadata->hType = set_compressed;
154
- nextHuf->repeatMode = HUF_repeat_check;
155
- return hSize;
156
- }
157
- }
158
- }
159
-
160
- /** ZSTD_buildSuperBlockEntropy_sequences() :
161
- * Builds entropy for the super-block sequences.
162
- * Stores symbol compression modes and fse table to fseMetadata.
163
- * @return : size of fse tables or error code */
164
- static size_t ZSTD_buildSuperBlockEntropy_sequences(seqStore_t* seqStorePtr,
165
- const ZSTD_fseCTables_t* prevEntropy,
166
- ZSTD_fseCTables_t* nextEntropy,
167
- const ZSTD_CCtx_params* cctxParams,
168
- ZSTD_fseCTablesMetadata_t* fseMetadata,
169
- void* workspace, size_t wkspSize)
170
- {
171
- BYTE* const wkspStart = (BYTE*)workspace;
172
- BYTE* const wkspEnd = wkspStart + wkspSize;
173
- BYTE* const countWkspStart = wkspStart;
174
- unsigned* const countWksp = (unsigned*)workspace;
175
- const size_t countWkspSize = (MaxSeq + 1) * sizeof(unsigned);
176
- BYTE* const cTableWksp = countWkspStart + countWkspSize;
177
- const size_t cTableWkspSize = wkspEnd-cTableWksp;
178
- ZSTD_strategy const strategy = cctxParams->cParams.strategy;
179
- FSE_CTable* CTable_LitLength = nextEntropy->litlengthCTable;
180
- FSE_CTable* CTable_OffsetBits = nextEntropy->offcodeCTable;
181
- FSE_CTable* CTable_MatchLength = nextEntropy->matchlengthCTable;
182
- const BYTE* const ofCodeTable = seqStorePtr->ofCode;
183
- const BYTE* const llCodeTable = seqStorePtr->llCode;
184
- const BYTE* const mlCodeTable = seqStorePtr->mlCode;
185
- size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart;
186
- BYTE* const ostart = fseMetadata->fseTablesBuffer;
187
- BYTE* const oend = ostart + sizeof(fseMetadata->fseTablesBuffer);
188
- BYTE* op = ostart;
189
-
190
- assert(cTableWkspSize >= (1 << MaxFSELog) * sizeof(FSE_FUNCTION_TYPE));
191
- DEBUGLOG(5, "ZSTD_buildSuperBlockEntropy_sequences (nbSeq=%zu)", nbSeq);
192
- memset(workspace, 0, wkspSize);
193
-
194
- fseMetadata->lastCountSize = 0;
195
- /* convert length/distances into codes */
196
- ZSTD_seqToCodes(seqStorePtr);
197
- /* build CTable for Literal Lengths */
198
- { U32 LLtype;
199
- unsigned max = MaxLL;
200
- size_t const mostFrequent = HIST_countFast_wksp(countWksp, &max, llCodeTable, nbSeq, workspace, wkspSize); /* can't fail */
201
- DEBUGLOG(5, "Building LL table");
202
- nextEntropy->litlength_repeatMode = prevEntropy->litlength_repeatMode;
203
- LLtype = ZSTD_selectEncodingType(&nextEntropy->litlength_repeatMode,
204
- countWksp, max, mostFrequent, nbSeq,
205
- LLFSELog, prevEntropy->litlengthCTable,
206
- ZSTDInternalConstants::LL_defaultNorm, ZSTDInternalConstants::LL_defaultNormLog,
207
- ZSTD_defaultAllowed, strategy);
208
- assert(set_basic < set_compressed && set_rle < set_compressed);
209
- assert(!(LLtype < set_compressed && nextEntropy->litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */
210
- { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype,
211
- countWksp, max, llCodeTable, nbSeq, ZSTDInternalConstants::LL_defaultNorm, ZSTDInternalConstants::LL_defaultNormLog, MaxLL,
212
- prevEntropy->litlengthCTable, sizeof(prevEntropy->litlengthCTable),
213
- cTableWksp, cTableWkspSize);
214
- FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for LitLens failed");
215
- if (LLtype == set_compressed)
216
- fseMetadata->lastCountSize = countSize;
217
- op += countSize;
218
- fseMetadata->llType = (symbolEncodingType_e) LLtype;
219
- } }
220
- /* build CTable for Offsets */
221
- { U32 Offtype;
222
- unsigned max = MaxOff;
223
- size_t const mostFrequent = HIST_countFast_wksp(countWksp, &max, ofCodeTable, nbSeq, workspace, wkspSize); /* can't fail */
224
- /* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */
225
- ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed;
226
- DEBUGLOG(5, "Building OF table");
227
- nextEntropy->offcode_repeatMode = prevEntropy->offcode_repeatMode;
228
- Offtype = ZSTD_selectEncodingType(&nextEntropy->offcode_repeatMode,
229
- countWksp, max, mostFrequent, nbSeq,
230
- OffFSELog, prevEntropy->offcodeCTable,
231
- ZSTDInternalConstants::OF_defaultNorm, ZSTDInternalConstants::OF_defaultNormLog,
232
- defaultPolicy, strategy);
233
- assert(!(Offtype < set_compressed && nextEntropy->offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */
234
- { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype,
235
- countWksp, max, ofCodeTable, nbSeq, ZSTDInternalConstants::OF_defaultNorm, ZSTDInternalConstants::OF_defaultNormLog, DefaultMaxOff,
236
- prevEntropy->offcodeCTable, sizeof(prevEntropy->offcodeCTable),
237
- cTableWksp, cTableWkspSize);
238
- FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for Offsets failed");
239
- if (Offtype == set_compressed)
240
- fseMetadata->lastCountSize = countSize;
241
- op += countSize;
242
- fseMetadata->ofType = (symbolEncodingType_e) Offtype;
243
- } }
244
- /* build CTable for MatchLengths */
245
- { U32 MLtype;
246
- unsigned max = MaxML;
247
- size_t const mostFrequent = HIST_countFast_wksp(countWksp, &max, mlCodeTable, nbSeq, workspace, wkspSize); /* can't fail */
248
- DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op));
249
- nextEntropy->matchlength_repeatMode = prevEntropy->matchlength_repeatMode;
250
- MLtype = ZSTD_selectEncodingType(&nextEntropy->matchlength_repeatMode,
251
- countWksp, max, mostFrequent, nbSeq,
252
- MLFSELog, prevEntropy->matchlengthCTable,
253
- ZSTDInternalConstants::ML_defaultNorm, ZSTDInternalConstants::ML_defaultNormLog,
254
- ZSTD_defaultAllowed, strategy);
255
- assert(!(MLtype < set_compressed && nextEntropy->matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */
256
- { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype,
257
- countWksp, max, mlCodeTable, nbSeq, ZSTDInternalConstants::ML_defaultNorm, ZSTDInternalConstants::ML_defaultNormLog, MaxML,
258
- prevEntropy->matchlengthCTable, sizeof(prevEntropy->matchlengthCTable),
259
- cTableWksp, cTableWkspSize);
260
- FORWARD_IF_ERROR(countSize, "ZSTD_buildCTable for MatchLengths failed");
261
- if (MLtype == set_compressed)
262
- fseMetadata->lastCountSize = countSize;
263
- op += countSize;
264
- fseMetadata->mlType = (symbolEncodingType_e) MLtype;
265
- } }
266
- assert((size_t) (op-ostart) <= sizeof(fseMetadata->fseTablesBuffer));
267
- return op-ostart;
268
- }
269
-
270
-
271
- /** ZSTD_buildSuperBlockEntropy() :
272
- * Builds entropy for the super-block.
273
- * @return : 0 on success or error code */
274
- static size_t
275
- ZSTD_buildSuperBlockEntropy(seqStore_t* seqStorePtr,
276
- const ZSTD_entropyCTables_t* prevEntropy,
277
- ZSTD_entropyCTables_t* nextEntropy,
278
- const ZSTD_CCtx_params* cctxParams,
279
- ZSTD_entropyCTablesMetadata_t* entropyMetadata,
280
- void* workspace, size_t wkspSize)
281
- {
282
- size_t const litSize = seqStorePtr->lit - seqStorePtr->litStart;
283
- DEBUGLOG(5, "ZSTD_buildSuperBlockEntropy");
284
- entropyMetadata->hufMetadata.hufDesSize =
285
- ZSTD_buildSuperBlockEntropy_literal(seqStorePtr->litStart, litSize,
286
- &prevEntropy->huf, &nextEntropy->huf,
287
- &entropyMetadata->hufMetadata,
288
- ZSTD_disableLiteralsCompression(cctxParams),
289
- workspace, wkspSize);
290
- FORWARD_IF_ERROR(entropyMetadata->hufMetadata.hufDesSize, "ZSTD_buildSuperBlockEntropy_literal failed");
291
- entropyMetadata->fseMetadata.fseTablesSize =
292
- ZSTD_buildSuperBlockEntropy_sequences(seqStorePtr,
293
- &prevEntropy->fse, &nextEntropy->fse,
294
- cctxParams,
295
- &entropyMetadata->fseMetadata,
296
- workspace, wkspSize);
297
- FORWARD_IF_ERROR(entropyMetadata->fseMetadata.fseTablesSize, "ZSTD_buildSuperBlockEntropy_sequences failed");
298
- return 0;
299
- }
300
23
 
301
24
  /** ZSTD_compressSubBlock_literal() :
302
25
  * Compresses literals section for a sub-block.
@@ -305,7 +28,7 @@ ZSTD_buildSuperBlockEntropy(seqStore_t* seqStorePtr,
305
28
  * before we know the table size + compressed size, so we have a bound on the
306
29
  * table size. If we guessed incorrectly, we fall back to uncompressed literals.
307
30
  *
308
- * We write the header when writeEntropy=1 and set entropyWrriten=1 when we succeeded
31
+ * We write the header when writeEntropy=1 and set entropyWritten=1 when we succeeded
309
32
  * in writing the header, otherwise it is set to 0.
310
33
  *
311
34
  * hufMetadata->hType has literals block type info.
@@ -315,13 +38,14 @@ ZSTD_buildSuperBlockEntropy(seqStore_t* seqStorePtr,
315
38
  * If it is set_compressed, first sub-block's literals section will be Treeless_Literals_Block
316
39
  * and the following sub-blocks' literals sections will be Treeless_Literals_Block.
317
40
  * @return : compressed size of literals section of a sub-block
318
- * Or 0 if it unable to compress.
41
+ * Or 0 if unable to compress.
319
42
  * Or error code */
320
- static size_t ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
321
- const ZSTD_hufCTablesMetadata_t* hufMetadata,
322
- const BYTE* literals, size_t litSize,
323
- void* dst, size_t dstSize,
324
- const int bmi2, int writeEntropy, int* entropyWritten)
43
+ static size_t
44
+ ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
45
+ const ZSTD_hufCTablesMetadata_t* hufMetadata,
46
+ const BYTE* literals, size_t litSize,
47
+ void* dst, size_t dstSize,
48
+ const int bmi2, int writeEntropy, int* entropyWritten)
325
49
  {
326
50
  size_t const header = writeEntropy ? 200 : 0;
327
51
  size_t const lhSize = 3 + (litSize >= (1 KB - header)) + (litSize >= (16 KB - header));
@@ -332,8 +56,6 @@ static size_t ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
332
56
  symbolEncodingType_e hType = writeEntropy ? hufMetadata->hType : set_repeat;
333
57
  size_t cLitSize = 0;
334
58
 
335
- (void)bmi2; /* TODO bmi2... */
336
-
337
59
  DEBUGLOG(5, "ZSTD_compressSubBlock_literal (litSize=%zu, lhSize=%zu, writeEntropy=%d)", litSize, lhSize, writeEntropy);
338
60
 
339
61
  *entropyWritten = 0;
@@ -349,15 +71,15 @@ static size_t ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
349
71
  assert(hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat);
350
72
 
351
73
  if (writeEntropy && hufMetadata->hType == set_compressed) {
352
- memcpy(op, hufMetadata->hufDesBuffer, hufMetadata->hufDesSize);
74
+ ZSTD_memcpy(op, hufMetadata->hufDesBuffer, hufMetadata->hufDesSize);
353
75
  op += hufMetadata->hufDesSize;
354
76
  cLitSize += hufMetadata->hufDesSize;
355
77
  DEBUGLOG(5, "ZSTD_compressSubBlock_literal (hSize=%zu)", hufMetadata->hufDesSize);
356
78
  }
357
79
 
358
- /* TODO bmi2 */
359
- { const size_t cSize = singleStream ? HUF_compress1X_usingCTable(op, oend-op, literals, litSize, hufTable)
360
- : HUF_compress4X_usingCTable(op, oend-op, literals, litSize, hufTable);
80
+ { int const flags = bmi2 ? HUF_flags_bmi2 : 0;
81
+ const size_t cSize = singleStream ? HUF_compress1X_usingCTable(op, (size_t)(oend-op), literals, litSize, hufTable, flags)
82
+ : HUF_compress4X_usingCTable(op, (size_t)(oend-op), literals, litSize, hufTable, flags);
361
83
  op += cSize;
362
84
  cLitSize += cSize;
363
85
  if (cSize == 0 || ERR_isError(cSize)) {
@@ -382,7 +104,7 @@ static size_t ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
382
104
  switch(lhSize)
383
105
  {
384
106
  case 3: /* 2 - 2 - 10 - 10 */
385
- { U32 const lhc = hType + ((!singleStream) << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<14);
107
+ { U32 const lhc = hType + ((U32)(!singleStream) << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<14);
386
108
  MEM_writeLE24(ostart, lhc);
387
109
  break;
388
110
  }
@@ -402,19 +124,30 @@ static size_t ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable,
402
124
  }
403
125
  *entropyWritten = 1;
404
126
  DEBUGLOG(5, "Compressed literals: %u -> %u", (U32)litSize, (U32)(op-ostart));
405
- return op-ostart;
127
+ return (size_t)(op-ostart);
406
128
  }
407
129
 
408
- static size_t ZSTD_seqDecompressedSize(seqStore_t const* seqStore, const seqDef* sequences, size_t nbSeq, size_t litSize, int lastSequence) {
409
- const seqDef* const sstart = sequences;
410
- const seqDef* const send = sequences + nbSeq;
411
- const seqDef* sp = sstart;
130
+ static size_t
131
+ ZSTD_seqDecompressedSize(seqStore_t const* seqStore,
132
+ const seqDef* sequences, size_t nbSeqs,
133
+ size_t litSize, int lastSubBlock)
134
+ {
412
135
  size_t matchLengthSum = 0;
413
- while (send-sp > 0) {
414
- ZSTD_sequenceLength const seqLen = ZSTD_getSequenceLength(seqStore, sp);
136
+ size_t litLengthSum = 0;
137
+ size_t n;
138
+ for (n=0; n<nbSeqs; n++) {
139
+ const ZSTD_sequenceLength seqLen = ZSTD_getSequenceLength(seqStore, sequences+n);
140
+ litLengthSum += seqLen.litLength;
415
141
  matchLengthSum += seqLen.matchLength;
416
- sp++;
417
142
  }
143
+ DEBUGLOG(5, "ZSTD_seqDecompressedSize: %u sequences from %p: %u literals + %u matchlength",
144
+ (unsigned)nbSeqs, (const void*)sequences,
145
+ (unsigned)litLengthSum, (unsigned)matchLengthSum);
146
+ if (!lastSubBlock)
147
+ assert(litLengthSum == litSize);
148
+ else
149
+ assert(litLengthSum <= litSize);
150
+ (void)litLengthSum;
418
151
  return matchLengthSum + litSize;
419
152
  }
420
153
 
@@ -428,13 +161,14 @@ static size_t ZSTD_seqDecompressedSize(seqStore_t const* seqStore, const seqDef*
428
161
  * @return : compressed size of sequences section of a sub-block
429
162
  * Or 0 if it is unable to compress
430
163
  * Or error code. */
431
- static size_t ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables,
432
- const ZSTD_fseCTablesMetadata_t* fseMetadata,
433
- const seqDef* sequences, size_t nbSeq,
434
- const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode,
435
- const ZSTD_CCtx_params* cctxParams,
436
- void* dst, size_t dstCapacity,
437
- const int bmi2, int writeEntropy, int* entropyWritten)
164
+ static size_t
165
+ ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables,
166
+ const ZSTD_fseCTablesMetadata_t* fseMetadata,
167
+ const seqDef* sequences, size_t nbSeq,
168
+ const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode,
169
+ const ZSTD_CCtx_params* cctxParams,
170
+ void* dst, size_t dstCapacity,
171
+ const int bmi2, int writeEntropy, int* entropyWritten)
438
172
  {
439
173
  const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN;
440
174
  BYTE* const ostart = (BYTE*)dst;
@@ -448,14 +182,14 @@ static size_t ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables
448
182
  /* Sequences Header */
449
183
  RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/,
450
184
  dstSize_tooSmall, "");
451
- if (nbSeq < 0x7F)
185
+ if (nbSeq < 128)
452
186
  *op++ = (BYTE)nbSeq;
453
187
  else if (nbSeq < LONGNBSEQ)
454
188
  op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2;
455
189
  else
456
190
  op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3;
457
191
  if (nbSeq==0) {
458
- return op - ostart;
192
+ return (size_t)(op - ostart);
459
193
  }
460
194
 
461
195
  /* seqHead : flags for FSE encoding type */
@@ -469,7 +203,7 @@ static size_t ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables
469
203
  const U32 MLtype = fseMetadata->mlType;
470
204
  DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (fseTablesSize=%zu)", fseMetadata->fseTablesSize);
471
205
  *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2));
472
- memcpy(op, fseMetadata->fseTablesBuffer, fseMetadata->fseTablesSize);
206
+ ZSTD_memcpy(op, fseMetadata->fseTablesBuffer, fseMetadata->fseTablesSize);
473
207
  op += fseMetadata->fseTablesSize;
474
208
  } else {
475
209
  const U32 repeat = set_repeat;
@@ -477,7 +211,7 @@ static size_t ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables
477
211
  }
478
212
 
479
213
  { size_t const bitstreamSize = ZSTD_encodeSequences(
480
- op, oend - op,
214
+ op, (size_t)(oend - op),
481
215
  fseTables->matchlengthCTable, mlCode,
482
216
  fseTables->offcodeCTable, ofCode,
483
217
  fseTables->litlengthCTable, llCode,
@@ -521,7 +255,7 @@ static size_t ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables
521
255
  #endif
522
256
 
523
257
  *entropyWritten = 1;
524
- return op - ostart;
258
+ return (size_t)(op - ostart);
525
259
  }
526
260
 
527
261
  /** ZSTD_compressSubBlock() :
@@ -542,12 +276,13 @@ static size_t ZSTD_compressSubBlock(const ZSTD_entropyCTables_t* entropy,
542
276
  {
543
277
  BYTE* const ostart = (BYTE*)dst;
544
278
  BYTE* const oend = ostart + dstCapacity;
545
- BYTE* op = ostart + ZSTDInternalConstants::ZSTD_blockHeaderSize;
279
+ BYTE* op = ostart + ZSTD_blockHeaderSize;
546
280
  DEBUGLOG(5, "ZSTD_compressSubBlock (litSize=%zu, nbSeq=%zu, writeLitEntropy=%d, writeSeqEntropy=%d, lastBlock=%d)",
547
281
  litSize, nbSeq, writeLitEntropy, writeSeqEntropy, lastBlock);
548
282
  { size_t cLitSize = ZSTD_compressSubBlock_literal((const HUF_CElt*)entropy->huf.CTable,
549
283
  &entropyMetadata->hufMetadata, literals, litSize,
550
- op, oend-op, bmi2, writeLitEntropy, litEntropyWritten);
284
+ op, (size_t)(oend-op),
285
+ bmi2, writeLitEntropy, litEntropyWritten);
551
286
  FORWARD_IF_ERROR(cLitSize, "ZSTD_compressSubBlock_literal failed");
552
287
  if (cLitSize == 0) return 0;
553
288
  op += cLitSize;
@@ -557,18 +292,18 @@ static size_t ZSTD_compressSubBlock(const ZSTD_entropyCTables_t* entropy,
557
292
  sequences, nbSeq,
558
293
  llCode, mlCode, ofCode,
559
294
  cctxParams,
560
- op, oend-op,
295
+ op, (size_t)(oend-op),
561
296
  bmi2, writeSeqEntropy, seqEntropyWritten);
562
297
  FORWARD_IF_ERROR(cSeqSize, "ZSTD_compressSubBlock_sequences failed");
563
298
  if (cSeqSize == 0) return 0;
564
299
  op += cSeqSize;
565
300
  }
566
301
  /* Write block header */
567
- { size_t cSize = (op-ostart)-ZSTDInternalConstants::ZSTD_blockHeaderSize;
302
+ { size_t cSize = (size_t)(op-ostart) - ZSTD_blockHeaderSize;
568
303
  U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3);
569
304
  MEM_writeLE24(ostart, cBlockHeader24);
570
305
  }
571
- return op-ostart;
306
+ return (size_t)(op-ostart);
572
307
  }
573
308
 
574
309
  static size_t ZSTD_estimateSubBlockSize_literal(const BYTE* literals, size_t litSize,
@@ -597,8 +332,8 @@ static size_t ZSTD_estimateSubBlockSize_literal(const BYTE* literals, size_t lit
597
332
  static size_t ZSTD_estimateSubBlockSize_symbolType(symbolEncodingType_e type,
598
333
  const BYTE* codeTable, unsigned maxCode,
599
334
  size_t nbSeq, const FSE_CTable* fseCTable,
600
- const U32* additionalBits,
601
- short const* defaultNorm, U32 defaultNormLog,
335
+ const U8* additionalBits,
336
+ short const* defaultNorm, U32 defaultNormLog, U32 defaultMax,
602
337
  void* workspace, size_t wkspSize)
603
338
  {
604
339
  unsigned* const countWksp = (unsigned*)workspace;
@@ -610,7 +345,11 @@ static size_t ZSTD_estimateSubBlockSize_symbolType(symbolEncodingType_e type,
610
345
 
611
346
  HIST_countFast_wksp(countWksp, &max, codeTable, nbSeq, workspace, wkspSize); /* can't fail */
612
347
  if (type == set_basic) {
613
- cSymbolTypeSizeEstimateInBits = ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, countWksp, max);
348
+ /* We selected this encoding type, so it must be valid. */
349
+ assert(max <= defaultMax);
350
+ cSymbolTypeSizeEstimateInBits = max <= defaultMax
351
+ ? ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, countWksp, max)
352
+ : ERROR(GENERIC);
614
353
  } else if (type == set_rle) {
615
354
  cSymbolTypeSizeEstimateInBits = 0;
616
355
  } else if (type == set_compressed || type == set_repeat) {
@@ -634,25 +373,30 @@ static size_t ZSTD_estimateSubBlockSize_sequences(const BYTE* ofCodeTable,
634
373
  void* workspace, size_t wkspSize,
635
374
  int writeEntropy)
636
375
  {
637
- size_t sequencesSectionHeaderSize = 3; /* Use hard coded size of 3 bytes */
376
+ size_t const sequencesSectionHeaderSize = 3; /* Use hard coded size of 3 bytes */
638
377
  size_t cSeqSizeEstimate = 0;
378
+ if (nbSeq == 0) return sequencesSectionHeaderSize;
639
379
  cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, MaxOff,
640
380
  nbSeq, fseTables->offcodeCTable, NULL,
641
- ZSTDInternalConstants::OF_defaultNorm, ZSTDInternalConstants::OF_defaultNormLog,
381
+ OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff,
642
382
  workspace, wkspSize);
643
383
  cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->llType, llCodeTable, MaxLL,
644
- nbSeq, fseTables->litlengthCTable, ZSTDInternalConstants::LL_bits,
645
- ZSTDInternalConstants::LL_defaultNorm, ZSTDInternalConstants::LL_defaultNormLog,
384
+ nbSeq, fseTables->litlengthCTable, LL_bits,
385
+ LL_defaultNorm, LL_defaultNormLog, MaxLL,
646
386
  workspace, wkspSize);
647
387
  cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->mlType, mlCodeTable, MaxML,
648
- nbSeq, fseTables->matchlengthCTable, ZSTDInternalConstants::ML_bits,
649
- ZSTDInternalConstants::ML_defaultNorm, ZSTDInternalConstants::ML_defaultNormLog,
388
+ nbSeq, fseTables->matchlengthCTable, ML_bits,
389
+ ML_defaultNorm, ML_defaultNormLog, MaxML,
650
390
  workspace, wkspSize);
651
391
  if (writeEntropy) cSeqSizeEstimate += fseMetadata->fseTablesSize;
652
392
  return cSeqSizeEstimate + sequencesSectionHeaderSize;
653
393
  }
654
394
 
655
- static size_t ZSTD_estimateSubBlockSize(const BYTE* literals, size_t litSize,
395
+ typedef struct {
396
+ size_t estLitSize;
397
+ size_t estBlockSize;
398
+ } EstimatedBlockSize;
399
+ static EstimatedBlockSize ZSTD_estimateSubBlockSize(const BYTE* literals, size_t litSize,
656
400
  const BYTE* ofCodeTable,
657
401
  const BYTE* llCodeTable,
658
402
  const BYTE* mlCodeTable,
@@ -660,15 +404,17 @@ static size_t ZSTD_estimateSubBlockSize(const BYTE* literals, size_t litSize,
660
404
  const ZSTD_entropyCTables_t* entropy,
661
405
  const ZSTD_entropyCTablesMetadata_t* entropyMetadata,
662
406
  void* workspace, size_t wkspSize,
663
- int writeLitEntropy, int writeSeqEntropy) {
664
- size_t cSizeEstimate = 0;
665
- cSizeEstimate += ZSTD_estimateSubBlockSize_literal(literals, litSize,
666
- &entropy->huf, &entropyMetadata->hufMetadata,
667
- workspace, wkspSize, writeLitEntropy);
668
- cSizeEstimate += ZSTD_estimateSubBlockSize_sequences(ofCodeTable, llCodeTable, mlCodeTable,
407
+ int writeLitEntropy, int writeSeqEntropy)
408
+ {
409
+ EstimatedBlockSize ebs;
410
+ ebs.estLitSize = ZSTD_estimateSubBlockSize_literal(literals, litSize,
411
+ &entropy->huf, &entropyMetadata->hufMetadata,
412
+ workspace, wkspSize, writeLitEntropy);
413
+ ebs.estBlockSize = ZSTD_estimateSubBlockSize_sequences(ofCodeTable, llCodeTable, mlCodeTable,
669
414
  nbSeq, &entropy->fse, &entropyMetadata->fseMetadata,
670
415
  workspace, wkspSize, writeSeqEntropy);
671
- return cSizeEstimate + ZSTDInternalConstants::ZSTD_blockHeaderSize;
416
+ ebs.estBlockSize += ebs.estLitSize + ZSTD_blockHeaderSize;
417
+ return ebs;
672
418
  }
673
419
 
674
420
  static int ZSTD_needSequenceEntropyTables(ZSTD_fseCTablesMetadata_t const* fseMetadata)
@@ -682,13 +428,56 @@ static int ZSTD_needSequenceEntropyTables(ZSTD_fseCTablesMetadata_t const* fseMe
682
428
  return 0;
683
429
  }
684
430
 
431
+ static size_t countLiterals(seqStore_t const* seqStore, const seqDef* sp, size_t seqCount)
432
+ {
433
+ size_t n, total = 0;
434
+ assert(sp != NULL);
435
+ for (n=0; n<seqCount; n++) {
436
+ total += ZSTD_getSequenceLength(seqStore, sp+n).litLength;
437
+ }
438
+ DEBUGLOG(6, "countLiterals for %zu sequences from %p => %zu bytes", seqCount, (const void*)sp, total);
439
+ return total;
440
+ }
441
+
442
+ #define BYTESCALE 256
443
+
444
+ static size_t sizeBlockSequences(const seqDef* sp, size_t nbSeqs,
445
+ size_t targetBudget, size_t avgLitCost, size_t avgSeqCost,
446
+ int firstSubBlock)
447
+ {
448
+ size_t n, budget = 0, inSize=0;
449
+ /* entropy headers */
450
+ size_t const headerSize = (size_t)firstSubBlock * 120 * BYTESCALE; /* generous estimate */
451
+ assert(firstSubBlock==0 || firstSubBlock==1);
452
+ budget += headerSize;
453
+
454
+ /* first sequence => at least one sequence*/
455
+ budget += sp[0].litLength * avgLitCost + avgSeqCost;
456
+ if (budget > targetBudget) return 1;
457
+ inSize = sp[0].litLength + (sp[0].mlBase+MINMATCH);
458
+
459
+ /* loop over sequences */
460
+ for (n=1; n<nbSeqs; n++) {
461
+ size_t currentCost = sp[n].litLength * avgLitCost + avgSeqCost;
462
+ budget += currentCost;
463
+ inSize += sp[n].litLength + (sp[n].mlBase+MINMATCH);
464
+ /* stop when sub-block budget is reached */
465
+ if ( (budget > targetBudget)
466
+ /* though continue to expand until the sub-block is deemed compressible */
467
+ && (budget < inSize * BYTESCALE) )
468
+ break;
469
+ }
470
+
471
+ return n;
472
+ }
473
+
685
474
  /** ZSTD_compressSubBlock_multi() :
686
475
  * Breaks super-block into multiple sub-blocks and compresses them.
687
- * Entropy will be written to the first block.
688
- * The following blocks will use repeat mode to compress.
689
- * All sub-blocks are compressed blocks (no raw or rle blocks).
690
- * @return : compressed size of the super block (which is multiple ZSTD blocks)
691
- * Or 0 if it failed to compress. */
476
+ * Entropy will be written into the first block.
477
+ * The following blocks use repeat_mode to compress.
478
+ * Sub-blocks are all compressed, except the last one when beneficial.
479
+ * @return : compressed size of the super block (which features multiple ZSTD blocks)
480
+ * or 0 if it failed to compress. */
692
481
  static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr,
693
482
  const ZSTD_compressedBlockState_t* prevCBlock,
694
483
  ZSTD_compressedBlockState_t* nextCBlock,
@@ -701,10 +490,12 @@ static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr,
701
490
  {
702
491
  const seqDef* const sstart = seqStorePtr->sequencesStart;
703
492
  const seqDef* const send = seqStorePtr->sequences;
704
- const seqDef* sp = sstart;
493
+ const seqDef* sp = sstart; /* tracks progresses within seqStorePtr->sequences */
494
+ size_t const nbSeqs = (size_t)(send - sstart);
705
495
  const BYTE* const lstart = seqStorePtr->litStart;
706
496
  const BYTE* const lend = seqStorePtr->lit;
707
497
  const BYTE* lp = lstart;
498
+ size_t const nbLiterals = (size_t)(lend - lstart);
708
499
  BYTE const* ip = (BYTE const*)src;
709
500
  BYTE const* const iend = ip + srcSize;
710
501
  BYTE* const ostart = (BYTE*)dst;
@@ -713,120 +504,179 @@ static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr,
713
504
  const BYTE* llCodePtr = seqStorePtr->llCode;
714
505
  const BYTE* mlCodePtr = seqStorePtr->mlCode;
715
506
  const BYTE* ofCodePtr = seqStorePtr->ofCode;
716
- size_t targetCBlockSize = cctxParams->targetCBlockSize;
717
- size_t litSize, seqCount;
718
- int writeLitEntropy = entropyMetadata->hufMetadata.hType == set_compressed;
507
+ size_t const minTarget = ZSTD_TARGETCBLOCKSIZE_MIN; /* enforce minimum size, to reduce undesirable side effects */
508
+ size_t const targetCBlockSize = MAX(minTarget, cctxParams->targetCBlockSize);
509
+ int writeLitEntropy = (entropyMetadata->hufMetadata.hType == set_compressed);
719
510
  int writeSeqEntropy = 1;
720
- int lastSequence = 0;
721
-
722
- DEBUGLOG(5, "ZSTD_compressSubBlock_multi (litSize=%u, nbSeq=%u)",
723
- (unsigned)(lend-lp), (unsigned)(send-sstart));
724
-
725
- litSize = 0;
726
- seqCount = 0;
727
- do {
728
- size_t cBlockSizeEstimate = 0;
729
- if (sstart == send) {
730
- lastSequence = 1;
731
- } else {
732
- const seqDef* const sequence = sp + seqCount;
733
- lastSequence = sequence == send - 1;
734
- litSize += ZSTD_getSequenceLength(seqStorePtr, sequence).litLength;
735
- seqCount++;
736
- }
737
- if (lastSequence) {
738
- assert(lp <= lend);
739
- assert(litSize <= (size_t)(lend - lp));
740
- litSize = (size_t)(lend - lp);
511
+
512
+ DEBUGLOG(5, "ZSTD_compressSubBlock_multi (srcSize=%u, litSize=%u, nbSeq=%u)",
513
+ (unsigned)srcSize, (unsigned)(lend-lstart), (unsigned)(send-sstart));
514
+
515
+ /* let's start by a general estimation for the full block */
516
+ if (nbSeqs > 0) {
517
+ EstimatedBlockSize const ebs =
518
+ ZSTD_estimateSubBlockSize(lp, nbLiterals,
519
+ ofCodePtr, llCodePtr, mlCodePtr, nbSeqs,
520
+ &nextCBlock->entropy, entropyMetadata,
521
+ workspace, wkspSize,
522
+ writeLitEntropy, writeSeqEntropy);
523
+ /* quick estimation */
524
+ size_t const avgLitCost = nbLiterals ? (ebs.estLitSize * BYTESCALE) / nbLiterals : BYTESCALE;
525
+ size_t const avgSeqCost = ((ebs.estBlockSize - ebs.estLitSize) * BYTESCALE) / nbSeqs;
526
+ const size_t nbSubBlocks = MAX((ebs.estBlockSize + (targetCBlockSize/2)) / targetCBlockSize, 1);
527
+ size_t n, avgBlockBudget, blockBudgetSupp=0;
528
+ avgBlockBudget = (ebs.estBlockSize * BYTESCALE) / nbSubBlocks;
529
+ DEBUGLOG(5, "estimated fullblock size=%u bytes ; avgLitCost=%.2f ; avgSeqCost=%.2f ; targetCBlockSize=%u, nbSubBlocks=%u ; avgBlockBudget=%.0f bytes",
530
+ (unsigned)ebs.estBlockSize, (double)avgLitCost/BYTESCALE, (double)avgSeqCost/BYTESCALE,
531
+ (unsigned)targetCBlockSize, (unsigned)nbSubBlocks, (double)avgBlockBudget/BYTESCALE);
532
+ /* simplification: if estimates states that the full superblock doesn't compress, just bail out immediately
533
+ * this will result in the production of a single uncompressed block covering @srcSize.*/
534
+ if (ebs.estBlockSize > srcSize) return 0;
535
+
536
+ /* compress and write sub-blocks */
537
+ assert(nbSubBlocks>0);
538
+ for (n=0; n < nbSubBlocks-1; n++) {
539
+ /* determine nb of sequences for current sub-block + nbLiterals from next sequence */
540
+ size_t const seqCount = sizeBlockSequences(sp, (size_t)(send-sp),
541
+ avgBlockBudget + blockBudgetSupp, avgLitCost, avgSeqCost, n==0);
542
+ /* if reached last sequence : break to last sub-block (simplification) */
543
+ assert(seqCount <= (size_t)(send-sp));
544
+ if (sp + seqCount == send) break;
545
+ assert(seqCount > 0);
546
+ /* compress sub-block */
547
+ { int litEntropyWritten = 0;
548
+ int seqEntropyWritten = 0;
549
+ size_t litSize = countLiterals(seqStorePtr, sp, seqCount);
550
+ const size_t decompressedSize =
551
+ ZSTD_seqDecompressedSize(seqStorePtr, sp, seqCount, litSize, 0);
552
+ size_t const cSize = ZSTD_compressSubBlock(&nextCBlock->entropy, entropyMetadata,
553
+ sp, seqCount,
554
+ lp, litSize,
555
+ llCodePtr, mlCodePtr, ofCodePtr,
556
+ cctxParams,
557
+ op, (size_t)(oend-op),
558
+ bmi2, writeLitEntropy, writeSeqEntropy,
559
+ &litEntropyWritten, &seqEntropyWritten,
560
+ 0);
561
+ FORWARD_IF_ERROR(cSize, "ZSTD_compressSubBlock failed");
562
+
563
+ /* check compressibility, update state components */
564
+ if (cSize > 0 && cSize < decompressedSize) {
565
+ DEBUGLOG(5, "Committed sub-block compressing %u bytes => %u bytes",
566
+ (unsigned)decompressedSize, (unsigned)cSize);
567
+ assert(ip + decompressedSize <= iend);
568
+ ip += decompressedSize;
569
+ lp += litSize;
570
+ op += cSize;
571
+ llCodePtr += seqCount;
572
+ mlCodePtr += seqCount;
573
+ ofCodePtr += seqCount;
574
+ /* Entropy only needs to be written once */
575
+ if (litEntropyWritten) {
576
+ writeLitEntropy = 0;
577
+ }
578
+ if (seqEntropyWritten) {
579
+ writeSeqEntropy = 0;
580
+ }
581
+ sp += seqCount;
582
+ blockBudgetSupp = 0;
583
+ } }
584
+ /* otherwise : do not compress yet, coalesce current sub-block with following one */
741
585
  }
742
- /* I think there is an optimization opportunity here.
743
- * Calling ZSTD_estimateSubBlockSize for every sequence can be wasteful
744
- * since it recalculates estimate from scratch.
745
- * For example, it would recount literal distribution and symbol codes everytime.
746
- */
747
- cBlockSizeEstimate = ZSTD_estimateSubBlockSize(lp, litSize, ofCodePtr, llCodePtr, mlCodePtr, seqCount,
748
- &nextCBlock->entropy, entropyMetadata,
749
- workspace, wkspSize, writeLitEntropy, writeSeqEntropy);
750
- if (cBlockSizeEstimate > targetCBlockSize || lastSequence) {
751
- int litEntropyWritten = 0;
752
- int seqEntropyWritten = 0;
753
- const size_t decompressedSize = ZSTD_seqDecompressedSize(seqStorePtr, sp, seqCount, litSize, lastSequence);
754
- const size_t cSize = ZSTD_compressSubBlock(&nextCBlock->entropy, entropyMetadata,
755
- sp, seqCount,
756
- lp, litSize,
757
- llCodePtr, mlCodePtr, ofCodePtr,
758
- cctxParams,
759
- op, oend-op,
760
- bmi2, writeLitEntropy, writeSeqEntropy,
761
- &litEntropyWritten, &seqEntropyWritten,
762
- lastBlock && lastSequence);
763
- FORWARD_IF_ERROR(cSize, "ZSTD_compressSubBlock failed");
764
- if (cSize > 0 && cSize < decompressedSize) {
765
- DEBUGLOG(5, "Committed the sub-block");
766
- assert(ip + decompressedSize <= iend);
767
- ip += decompressedSize;
768
- sp += seqCount;
769
- lp += litSize;
770
- op += cSize;
771
- llCodePtr += seqCount;
772
- mlCodePtr += seqCount;
773
- ofCodePtr += seqCount;
774
- litSize = 0;
775
- seqCount = 0;
776
- /* Entropy only needs to be written once */
777
- if (litEntropyWritten) {
778
- writeLitEntropy = 0;
779
- }
780
- if (seqEntropyWritten) {
781
- writeSeqEntropy = 0;
782
- }
586
+ } /* if (nbSeqs > 0) */
587
+
588
+ /* write last block */
589
+ DEBUGLOG(5, "Generate last sub-block: %u sequences remaining", (unsigned)(send - sp));
590
+ { int litEntropyWritten = 0;
591
+ int seqEntropyWritten = 0;
592
+ size_t litSize = (size_t)(lend - lp);
593
+ size_t seqCount = (size_t)(send - sp);
594
+ const size_t decompressedSize =
595
+ ZSTD_seqDecompressedSize(seqStorePtr, sp, seqCount, litSize, 1);
596
+ size_t const cSize = ZSTD_compressSubBlock(&nextCBlock->entropy, entropyMetadata,
597
+ sp, seqCount,
598
+ lp, litSize,
599
+ llCodePtr, mlCodePtr, ofCodePtr,
600
+ cctxParams,
601
+ op, (size_t)(oend-op),
602
+ bmi2, writeLitEntropy, writeSeqEntropy,
603
+ &litEntropyWritten, &seqEntropyWritten,
604
+ lastBlock);
605
+ FORWARD_IF_ERROR(cSize, "ZSTD_compressSubBlock failed");
606
+
607
+ /* update pointers, the nb of literals borrowed from next sequence must be preserved */
608
+ if (cSize > 0 && cSize < decompressedSize) {
609
+ DEBUGLOG(5, "Last sub-block compressed %u bytes => %u bytes",
610
+ (unsigned)decompressedSize, (unsigned)cSize);
611
+ assert(ip + decompressedSize <= iend);
612
+ ip += decompressedSize;
613
+ lp += litSize;
614
+ op += cSize;
615
+ llCodePtr += seqCount;
616
+ mlCodePtr += seqCount;
617
+ ofCodePtr += seqCount;
618
+ /* Entropy only needs to be written once */
619
+ if (litEntropyWritten) {
620
+ writeLitEntropy = 0;
621
+ }
622
+ if (seqEntropyWritten) {
623
+ writeSeqEntropy = 0;
783
624
  }
625
+ sp += seqCount;
784
626
  }
785
- } while (!lastSequence);
627
+ }
628
+
629
+
786
630
  if (writeLitEntropy) {
787
- DEBUGLOG(5, "ZSTD_compressSubBlock_multi has literal entropy tables unwritten");
788
- memcpy(&nextCBlock->entropy.huf, &prevCBlock->entropy.huf, sizeof(prevCBlock->entropy.huf));
631
+ DEBUGLOG(5, "Literal entropy tables were never written");
632
+ ZSTD_memcpy(&nextCBlock->entropy.huf, &prevCBlock->entropy.huf, sizeof(prevCBlock->entropy.huf));
789
633
  }
790
634
  if (writeSeqEntropy && ZSTD_needSequenceEntropyTables(&entropyMetadata->fseMetadata)) {
791
635
  /* If we haven't written our entropy tables, then we've violated our contract and
792
636
  * must emit an uncompressed block.
793
637
  */
794
- DEBUGLOG(5, "ZSTD_compressSubBlock_multi has sequence entropy tables unwritten");
638
+ DEBUGLOG(5, "Sequence entropy tables were never written => cancel, emit an uncompressed block");
795
639
  return 0;
796
640
  }
641
+
797
642
  if (ip < iend) {
798
- size_t const cSize = ZSTD_noCompressBlock(op, oend - op, ip, iend - ip, lastBlock);
799
- DEBUGLOG(5, "ZSTD_compressSubBlock_multi last sub-block uncompressed, %zu bytes", (size_t)(iend - ip));
643
+ /* some data left : last part of the block sent uncompressed */
644
+ size_t const rSize = (size_t)((iend - ip));
645
+ size_t const cSize = ZSTD_noCompressBlock(op, (size_t)(oend - op), ip, rSize, lastBlock);
646
+ DEBUGLOG(5, "Generate last uncompressed sub-block of %u bytes", (unsigned)(rSize));
800
647
  FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed");
801
648
  assert(cSize != 0);
802
649
  op += cSize;
803
650
  /* We have to regenerate the repcodes because we've skipped some sequences */
804
651
  if (sp < send) {
805
- seqDef const* seq;
652
+ const seqDef* seq;
806
653
  repcodes_t rep;
807
- memcpy(&rep, prevCBlock->rep, sizeof(rep));
654
+ ZSTD_memcpy(&rep, prevCBlock->rep, sizeof(rep));
808
655
  for (seq = sstart; seq < sp; ++seq) {
809
- rep = ZSTD_updateRep(rep.rep, seq->offset - 1, ZSTD_getSequenceLength(seqStorePtr, seq).litLength == 0);
656
+ ZSTD_updateRep(rep.rep, seq->offBase, ZSTD_getSequenceLength(seqStorePtr, seq).litLength == 0);
810
657
  }
811
- memcpy(nextCBlock->rep, &rep, sizeof(rep));
658
+ ZSTD_memcpy(nextCBlock->rep, &rep, sizeof(rep));
812
659
  }
813
660
  }
814
- DEBUGLOG(5, "ZSTD_compressSubBlock_multi compressed");
815
- return op-ostart;
661
+
662
+ DEBUGLOG(5, "ZSTD_compressSubBlock_multi compressed all subBlocks: total compressed size = %u",
663
+ (unsigned)(op-ostart));
664
+ return (size_t)(op-ostart);
816
665
  }
817
666
 
818
667
  size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc,
819
668
  void* dst, size_t dstCapacity,
820
- void const* src, size_t srcSize,
821
- unsigned lastBlock) {
669
+ const void* src, size_t srcSize,
670
+ unsigned lastBlock)
671
+ {
822
672
  ZSTD_entropyCTablesMetadata_t entropyMetadata;
823
673
 
824
- FORWARD_IF_ERROR(ZSTD_buildSuperBlockEntropy(&zc->seqStore,
674
+ FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(&zc->seqStore,
825
675
  &zc->blockState.prevCBlock->entropy,
826
676
  &zc->blockState.nextCBlock->entropy,
827
677
  &zc->appliedParams,
828
678
  &entropyMetadata,
829
- zc->entropyWorkspace, HUF_WORKSPACE_SIZE /* statically allocated in resetCCtx */), "");
679
+ zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */), "");
830
680
 
831
681
  return ZSTD_compressSubBlock_multi(&zc->seqStore,
832
682
  zc->blockState.prevCBlock,
@@ -836,7 +686,7 @@ size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc,
836
686
  dst, dstCapacity,
837
687
  src, srcSize,
838
688
  zc->bmi2, lastBlock,
839
- zc->entropyWorkspace, HUF_WORKSPACE_SIZE /* statically allocated in resetCCtx */);
689
+ zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */);
840
690
  }
841
691
 
842
- }
692
+ } // namespace duckdb_zstd