duckdb 0.3.2-dev396.0 → 0.3.2-dev454.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/duckdb.cpp CHANGED
@@ -80621,6 +80621,11 @@ struct RLEFun {
80621
80621
  static bool TypeIsSupported(PhysicalType type);
80622
80622
  };
80623
80623
 
80624
+ struct BitpackingFun {
80625
+ static CompressionFunction GetFunction(PhysicalType type);
80626
+ static bool TypeIsSupported(PhysicalType type);
80627
+ };
80628
+
80624
80629
  } // namespace duckdb
80625
80630
 
80626
80631
 
@@ -80640,6 +80645,7 @@ static DefaultCompressionMethod internal_compression_methods[] = {
80640
80645
  {CompressionType::COMPRESSION_CONSTANT, ConstantFun::GetFunction, ConstantFun::TypeIsSupported},
80641
80646
  {CompressionType::COMPRESSION_UNCOMPRESSED, UncompressedFun::GetFunction, UncompressedFun::TypeIsSupported},
80642
80647
  {CompressionType::COMPRESSION_RLE, RLEFun::GetFunction, RLEFun::TypeIsSupported},
80648
+ {CompressionType::COMPRESSION_BITPACKING, BitpackingFun::GetFunction, BitpackingFun::TypeIsSupported},
80643
80649
  {CompressionType::COMPRESSION_AUTO, nullptr, nullptr}};
80644
80650
 
80645
80651
  static CompressionFunction *FindCompressionFunction(CompressionFunctionSet &set, CompressionType type,
@@ -80688,6 +80694,7 @@ vector<CompressionFunction *> DBConfig::GetCompressionFunctions(PhysicalType dat
80688
80694
  vector<CompressionFunction *> result;
80689
80695
  TryLoadCompression(*this, result, CompressionType::COMPRESSION_UNCOMPRESSED, data_type);
80690
80696
  TryLoadCompression(*this, result, CompressionType::COMPRESSION_RLE, data_type);
80697
+ TryLoadCompression(*this, result, CompressionType::COMPRESSION_BITPACKING, data_type);
80691
80698
  return result;
80692
80699
  }
80693
80700
 
@@ -159282,56 +159289,1170 @@ void PartialBlock::FlushToDisk(DatabaseInstance &db) {
159282
159289
  }
159283
159290
 
159284
159291
  } // namespace duckdb
159285
- //===----------------------------------------------------------------------===//
159286
- // DuckDB
159287
- //
159288
- // duckdb/storage/segment/uncompressed.hpp
159289
- //
159290
- //
159291
- //===----------------------------------------------------------------------===//
159292
159292
 
159293
159293
 
159294
+ // LICENSE_CHANGE_BEGIN
159295
+ // The following code up to LICENSE_CHANGE_END is subject to THIRD PARTY LICENSE #10
159296
+ // See the end of this file for a list
159297
+
159298
+ /**
159299
+ * This code is released under the
159300
+ * Apache License Version 2.0 http://www.apache.org/licenses/.
159301
+ *
159302
+ * (c) Daniel Lemire, http://lemire.me/en/
159303
+ */
159304
+
159294
159305
 
159295
159306
 
159307
+ // LICENSE_CHANGE_BEGIN
159308
+ // The following code up to LICENSE_CHANGE_END is subject to THIRD PARTY LICENSE #10
159309
+ // See the end of this file for a list
159296
159310
 
159297
- namespace duckdb {
159298
- class DatabaseInstance;
159311
+ /**
159312
+ * This code is released under the
159313
+ * Apache License Version 2.0 http://www.apache.org/licenses/.
159314
+ *
159315
+ * (c) Daniel Lemire, http://fastpforlib.me/en/
159316
+ */
159299
159317
 
159300
- struct UncompressedFunctions {
159301
- static unique_ptr<CompressionState> InitCompression(ColumnDataCheckpointer &checkpointer,
159302
- unique_ptr<AnalyzeState> state);
159303
- static void Compress(CompressionState &state_p, Vector &data, idx_t count);
159304
- static void FinalizeCompress(CompressionState &state_p);
159305
- static void EmptySkip(ColumnSegment &segment, ColumnScanState &state, idx_t skip_count) {
159318
+ #include <cinttypes>
159319
+ #include <string>
159320
+
159321
+ namespace duckdb_fastpforlib {
159322
+ namespace internal {
159323
+
159324
+ // Unpacks 8 uint8_t values
159325
+ void __fastunpack0(const uint8_t *__restrict in, uint8_t *__restrict out);
159326
+ void __fastunpack1(const uint8_t *__restrict in, uint8_t *__restrict out);
159327
+ void __fastunpack2(const uint8_t *__restrict in, uint8_t *__restrict out);
159328
+ void __fastunpack3(const uint8_t *__restrict in, uint8_t *__restrict out);
159329
+ void __fastunpack4(const uint8_t *__restrict in, uint8_t *__restrict out);
159330
+ void __fastunpack5(const uint8_t *__restrict in, uint8_t *__restrict out);
159331
+ void __fastunpack6(const uint8_t *__restrict in, uint8_t *__restrict out);
159332
+ void __fastunpack7(const uint8_t *__restrict in, uint8_t *__restrict out);
159333
+ void __fastunpack8(const uint8_t *__restrict in, uint8_t *__restrict out);
159334
+
159335
+ // Unpacks 16 uint16_t values
159336
+ void __fastunpack0(const uint16_t *__restrict in, uint16_t *__restrict out);
159337
+ void __fastunpack1(const uint16_t *__restrict in, uint16_t *__restrict out);
159338
+ void __fastunpack2(const uint16_t *__restrict in, uint16_t *__restrict out);
159339
+ void __fastunpack3(const uint16_t *__restrict in, uint16_t *__restrict out);
159340
+ void __fastunpack4(const uint16_t *__restrict in, uint16_t *__restrict out);
159341
+ void __fastunpack5(const uint16_t *__restrict in, uint16_t *__restrict out);
159342
+ void __fastunpack6(const uint16_t *__restrict in, uint16_t *__restrict out);
159343
+ void __fastunpack7(const uint16_t *__restrict in, uint16_t *__restrict out);
159344
+ void __fastunpack8(const uint16_t *__restrict in, uint16_t *__restrict out);
159345
+ void __fastunpack9(const uint16_t *__restrict in, uint16_t *__restrict out);
159346
+ void __fastunpack10(const uint16_t *__restrict in, uint16_t *__restrict out);
159347
+ void __fastunpack11(const uint16_t *__restrict in, uint16_t *__restrict out);
159348
+ void __fastunpack12(const uint16_t *__restrict in, uint16_t *__restrict out);
159349
+ void __fastunpack13(const uint16_t *__restrict in, uint16_t *__restrict out);
159350
+ void __fastunpack14(const uint16_t *__restrict in, uint16_t *__restrict out);
159351
+ void __fastunpack15(const uint16_t *__restrict in, uint16_t *__restrict out);
159352
+ void __fastunpack16(const uint16_t *__restrict in, uint16_t *__restrict out);
159353
+
159354
+ // Unpacks 32 uint32_t values
159355
+ void __fastunpack0(const uint32_t *__restrict in, uint32_t *__restrict out);
159356
+ void __fastunpack1(const uint32_t *__restrict in, uint32_t *__restrict out);
159357
+ void __fastunpack2(const uint32_t *__restrict in, uint32_t *__restrict out);
159358
+ void __fastunpack3(const uint32_t *__restrict in, uint32_t *__restrict out);
159359
+ void __fastunpack4(const uint32_t *__restrict in, uint32_t *__restrict out);
159360
+ void __fastunpack5(const uint32_t *__restrict in, uint32_t *__restrict out);
159361
+ void __fastunpack6(const uint32_t *__restrict in, uint32_t *__restrict out);
159362
+ void __fastunpack7(const uint32_t *__restrict in, uint32_t *__restrict out);
159363
+ void __fastunpack8(const uint32_t *__restrict in, uint32_t *__restrict out);
159364
+ void __fastunpack9(const uint32_t *__restrict in, uint32_t *__restrict out);
159365
+ void __fastunpack10(const uint32_t *__restrict in, uint32_t *__restrict out);
159366
+ void __fastunpack11(const uint32_t *__restrict in, uint32_t *__restrict out);
159367
+ void __fastunpack12(const uint32_t *__restrict in, uint32_t *__restrict out);
159368
+ void __fastunpack13(const uint32_t *__restrict in, uint32_t *__restrict out);
159369
+ void __fastunpack14(const uint32_t *__restrict in, uint32_t *__restrict out);
159370
+ void __fastunpack15(const uint32_t *__restrict in, uint32_t *__restrict out);
159371
+ void __fastunpack16(const uint32_t *__restrict in, uint32_t *__restrict out);
159372
+ void __fastunpack17(const uint32_t *__restrict in, uint32_t *__restrict out);
159373
+ void __fastunpack18(const uint32_t *__restrict in, uint32_t *__restrict out);
159374
+ void __fastunpack19(const uint32_t *__restrict in, uint32_t *__restrict out);
159375
+ void __fastunpack20(const uint32_t *__restrict in, uint32_t *__restrict out);
159376
+ void __fastunpack21(const uint32_t *__restrict in, uint32_t *__restrict out);
159377
+ void __fastunpack22(const uint32_t *__restrict in, uint32_t *__restrict out);
159378
+ void __fastunpack23(const uint32_t *__restrict in, uint32_t *__restrict out);
159379
+ void __fastunpack24(const uint32_t *__restrict in, uint32_t *__restrict out);
159380
+ void __fastunpack25(const uint32_t *__restrict in, uint32_t *__restrict out);
159381
+ void __fastunpack26(const uint32_t *__restrict in, uint32_t *__restrict out);
159382
+ void __fastunpack27(const uint32_t *__restrict in, uint32_t *__restrict out);
159383
+ void __fastunpack28(const uint32_t *__restrict in, uint32_t *__restrict out);
159384
+ void __fastunpack29(const uint32_t *__restrict in, uint32_t *__restrict out);
159385
+ void __fastunpack30(const uint32_t *__restrict in, uint32_t *__restrict out);
159386
+ void __fastunpack31(const uint32_t *__restrict in, uint32_t *__restrict out);
159387
+ void __fastunpack32(const uint32_t *__restrict in, uint32_t *__restrict out);
159388
+
159389
+ // Unpacks 32 uint64_t values
159390
+ void __fastunpack0(const uint32_t *__restrict in, uint64_t *__restrict out);
159391
+ void __fastunpack1(const uint32_t *__restrict in, uint64_t *__restrict out);
159392
+ void __fastunpack2(const uint32_t *__restrict in, uint64_t *__restrict out);
159393
+ void __fastunpack3(const uint32_t *__restrict in, uint64_t *__restrict out);
159394
+ void __fastunpack4(const uint32_t *__restrict in, uint64_t *__restrict out);
159395
+ void __fastunpack5(const uint32_t *__restrict in, uint64_t *__restrict out);
159396
+ void __fastunpack6(const uint32_t *__restrict in, uint64_t *__restrict out);
159397
+ void __fastunpack7(const uint32_t *__restrict in, uint64_t *__restrict out);
159398
+ void __fastunpack8(const uint32_t *__restrict in, uint64_t *__restrict out);
159399
+ void __fastunpack9(const uint32_t *__restrict in, uint64_t *__restrict out);
159400
+ void __fastunpack10(const uint32_t *__restrict in, uint64_t *__restrict out);
159401
+ void __fastunpack11(const uint32_t *__restrict in, uint64_t *__restrict out);
159402
+ void __fastunpack12(const uint32_t *__restrict in, uint64_t *__restrict out);
159403
+ void __fastunpack13(const uint32_t *__restrict in, uint64_t *__restrict out);
159404
+ void __fastunpack14(const uint32_t *__restrict in, uint64_t *__restrict out);
159405
+ void __fastunpack15(const uint32_t *__restrict in, uint64_t *__restrict out);
159406
+ void __fastunpack16(const uint32_t *__restrict in, uint64_t *__restrict out);
159407
+ void __fastunpack17(const uint32_t *__restrict in, uint64_t *__restrict out);
159408
+ void __fastunpack18(const uint32_t *__restrict in, uint64_t *__restrict out);
159409
+ void __fastunpack19(const uint32_t *__restrict in, uint64_t *__restrict out);
159410
+ void __fastunpack20(const uint32_t *__restrict in, uint64_t *__restrict out);
159411
+ void __fastunpack21(const uint32_t *__restrict in, uint64_t *__restrict out);
159412
+ void __fastunpack22(const uint32_t *__restrict in, uint64_t *__restrict out);
159413
+ void __fastunpack23(const uint32_t *__restrict in, uint64_t *__restrict out);
159414
+ void __fastunpack24(const uint32_t *__restrict in, uint64_t *__restrict out);
159415
+ void __fastunpack25(const uint32_t *__restrict in, uint64_t *__restrict out);
159416
+ void __fastunpack26(const uint32_t *__restrict in, uint64_t *__restrict out);
159417
+ void __fastunpack27(const uint32_t *__restrict in, uint64_t *__restrict out);
159418
+ void __fastunpack28(const uint32_t *__restrict in, uint64_t *__restrict out);
159419
+ void __fastunpack29(const uint32_t *__restrict in, uint64_t *__restrict out);
159420
+ void __fastunpack30(const uint32_t *__restrict in, uint64_t *__restrict out);
159421
+ void __fastunpack31(const uint32_t *__restrict in, uint64_t *__restrict out);
159422
+ void __fastunpack32(const uint32_t *__restrict in, uint64_t *__restrict out);
159423
+ void __fastunpack33(const uint32_t *__restrict in, uint64_t *__restrict out);
159424
+ void __fastunpack34(const uint32_t *__restrict in, uint64_t *__restrict out);
159425
+ void __fastunpack35(const uint32_t *__restrict in, uint64_t *__restrict out);
159426
+ void __fastunpack36(const uint32_t *__restrict in, uint64_t *__restrict out);
159427
+ void __fastunpack37(const uint32_t *__restrict in, uint64_t *__restrict out);
159428
+ void __fastunpack38(const uint32_t *__restrict in, uint64_t *__restrict out);
159429
+ void __fastunpack39(const uint32_t *__restrict in, uint64_t *__restrict out);
159430
+ void __fastunpack40(const uint32_t *__restrict in, uint64_t *__restrict out);
159431
+ void __fastunpack41(const uint32_t *__restrict in, uint64_t *__restrict out);
159432
+ void __fastunpack42(const uint32_t *__restrict in, uint64_t *__restrict out);
159433
+ void __fastunpack43(const uint32_t *__restrict in, uint64_t *__restrict out);
159434
+ void __fastunpack44(const uint32_t *__restrict in, uint64_t *__restrict out);
159435
+ void __fastunpack45(const uint32_t *__restrict in, uint64_t *__restrict out);
159436
+ void __fastunpack46(const uint32_t *__restrict in, uint64_t *__restrict out);
159437
+ void __fastunpack47(const uint32_t *__restrict in, uint64_t *__restrict out);
159438
+ void __fastunpack48(const uint32_t *__restrict in, uint64_t *__restrict out);
159439
+ void __fastunpack49(const uint32_t *__restrict in, uint64_t *__restrict out);
159440
+ void __fastunpack50(const uint32_t *__restrict in, uint64_t *__restrict out);
159441
+ void __fastunpack51(const uint32_t *__restrict in, uint64_t *__restrict out);
159442
+ void __fastunpack52(const uint32_t *__restrict in, uint64_t *__restrict out);
159443
+ void __fastunpack53(const uint32_t *__restrict in, uint64_t *__restrict out);
159444
+ void __fastunpack54(const uint32_t *__restrict in, uint64_t *__restrict out);
159445
+ void __fastunpack55(const uint32_t *__restrict in, uint64_t *__restrict out);
159446
+ void __fastunpack56(const uint32_t *__restrict in, uint64_t *__restrict out);
159447
+ void __fastunpack57(const uint32_t *__restrict in, uint64_t *__restrict out);
159448
+ void __fastunpack58(const uint32_t *__restrict in, uint64_t *__restrict out);
159449
+ void __fastunpack59(const uint32_t *__restrict in, uint64_t *__restrict out);
159450
+ void __fastunpack60(const uint32_t *__restrict in, uint64_t *__restrict out);
159451
+ void __fastunpack61(const uint32_t *__restrict in, uint64_t *__restrict out);
159452
+ void __fastunpack62(const uint32_t *__restrict in, uint64_t *__restrict out);
159453
+ void __fastunpack63(const uint32_t *__restrict in, uint64_t *__restrict out);
159454
+ void __fastunpack64(const uint32_t *__restrict in, uint64_t *__restrict out);
159455
+
159456
+ // Packs 8 int8_t values
159457
+ void __fastpack0(const uint8_t *__restrict in, uint8_t *__restrict out);
159458
+ void __fastpack1(const uint8_t *__restrict in, uint8_t *__restrict out);
159459
+ void __fastpack2(const uint8_t *__restrict in, uint8_t *__restrict out);
159460
+ void __fastpack3(const uint8_t *__restrict in, uint8_t *__restrict out);
159461
+ void __fastpack4(const uint8_t *__restrict in, uint8_t *__restrict out);
159462
+ void __fastpack5(const uint8_t *__restrict in, uint8_t *__restrict out);
159463
+ void __fastpack6(const uint8_t *__restrict in, uint8_t *__restrict out);
159464
+ void __fastpack7(const uint8_t *__restrict in, uint8_t *__restrict out);
159465
+ void __fastpack8(const uint8_t *__restrict in, uint8_t *__restrict out);
159466
+
159467
+ // Packs 16 int16_t values
159468
+ void __fastpack0(const uint16_t *__restrict in, uint16_t *__restrict out);
159469
+ void __fastpack1(const uint16_t *__restrict in, uint16_t *__restrict out);
159470
+ void __fastpack2(const uint16_t *__restrict in, uint16_t *__restrict out);
159471
+ void __fastpack3(const uint16_t *__restrict in, uint16_t *__restrict out);
159472
+ void __fastpack4(const uint16_t *__restrict in, uint16_t *__restrict out);
159473
+ void __fastpack5(const uint16_t *__restrict in, uint16_t *__restrict out);
159474
+ void __fastpack6(const uint16_t *__restrict in, uint16_t *__restrict out);
159475
+ void __fastpack7(const uint16_t *__restrict in, uint16_t *__restrict out);
159476
+ void __fastpack8(const uint16_t *__restrict in, uint16_t *__restrict out);
159477
+ void __fastpack9(const uint16_t *__restrict in, uint16_t *__restrict out);
159478
+ void __fastpack10(const uint16_t *__restrict in, uint16_t *__restrict out);
159479
+ void __fastpack11(const uint16_t *__restrict in, uint16_t *__restrict out);
159480
+ void __fastpack12(const uint16_t *__restrict in, uint16_t *__restrict out);
159481
+ void __fastpack13(const uint16_t *__restrict in, uint16_t *__restrict out);
159482
+ void __fastpack14(const uint16_t *__restrict in, uint16_t *__restrict out);
159483
+ void __fastpack15(const uint16_t *__restrict in, uint16_t *__restrict out);
159484
+ void __fastpack16(const uint16_t *__restrict in, uint16_t *__restrict out);
159485
+
159486
+ // Packs 32 int32_t values
159487
+ void __fastpack0(const uint32_t *__restrict in, uint32_t *__restrict out);
159488
+ void __fastpack1(const uint32_t *__restrict in, uint32_t *__restrict out);
159489
+ void __fastpack2(const uint32_t *__restrict in, uint32_t *__restrict out);
159490
+ void __fastpack3(const uint32_t *__restrict in, uint32_t *__restrict out);
159491
+ void __fastpack4(const uint32_t *__restrict in, uint32_t *__restrict out);
159492
+ void __fastpack5(const uint32_t *__restrict in, uint32_t *__restrict out);
159493
+ void __fastpack6(const uint32_t *__restrict in, uint32_t *__restrict out);
159494
+ void __fastpack7(const uint32_t *__restrict in, uint32_t *__restrict out);
159495
+ void __fastpack8(const uint32_t *__restrict in, uint32_t *__restrict out);
159496
+ void __fastpack9(const uint32_t *__restrict in, uint32_t *__restrict out);
159497
+ void __fastpack10(const uint32_t *__restrict in, uint32_t *__restrict out);
159498
+ void __fastpack11(const uint32_t *__restrict in, uint32_t *__restrict out);
159499
+ void __fastpack12(const uint32_t *__restrict in, uint32_t *__restrict out);
159500
+ void __fastpack13(const uint32_t *__restrict in, uint32_t *__restrict out);
159501
+ void __fastpack14(const uint32_t *__restrict in, uint32_t *__restrict out);
159502
+ void __fastpack15(const uint32_t *__restrict in, uint32_t *__restrict out);
159503
+ void __fastpack16(const uint32_t *__restrict in, uint32_t *__restrict out);
159504
+ void __fastpack17(const uint32_t *__restrict in, uint32_t *__restrict out);
159505
+ void __fastpack18(const uint32_t *__restrict in, uint32_t *__restrict out);
159506
+ void __fastpack19(const uint32_t *__restrict in, uint32_t *__restrict out);
159507
+ void __fastpack20(const uint32_t *__restrict in, uint32_t *__restrict out);
159508
+ void __fastpack21(const uint32_t *__restrict in, uint32_t *__restrict out);
159509
+ void __fastpack22(const uint32_t *__restrict in, uint32_t *__restrict out);
159510
+ void __fastpack23(const uint32_t *__restrict in, uint32_t *__restrict out);
159511
+ void __fastpack24(const uint32_t *__restrict in, uint32_t *__restrict out);
159512
+ void __fastpack25(const uint32_t *__restrict in, uint32_t *__restrict out);
159513
+ void __fastpack26(const uint32_t *__restrict in, uint32_t *__restrict out);
159514
+ void __fastpack27(const uint32_t *__restrict in, uint32_t *__restrict out);
159515
+ void __fastpack28(const uint32_t *__restrict in, uint32_t *__restrict out);
159516
+ void __fastpack29(const uint32_t *__restrict in, uint32_t *__restrict out);
159517
+ void __fastpack30(const uint32_t *__restrict in, uint32_t *__restrict out);
159518
+ void __fastpack31(const uint32_t *__restrict in, uint32_t *__restrict out);
159519
+ void __fastpack32(const uint32_t *__restrict in, uint32_t *__restrict out);
159520
+
159521
+ // Packs 32 int64_t values
159522
+ void __fastpack0(const uint64_t *__restrict in, uint32_t *__restrict out);
159523
+ void __fastpack1(const uint64_t *__restrict in, uint32_t *__restrict out);
159524
+ void __fastpack2(const uint64_t *__restrict in, uint32_t *__restrict out);
159525
+ void __fastpack3(const uint64_t *__restrict in, uint32_t *__restrict out);
159526
+ void __fastpack4(const uint64_t *__restrict in, uint32_t *__restrict out);
159527
+ void __fastpack5(const uint64_t *__restrict in, uint32_t *__restrict out);
159528
+ void __fastpack6(const uint64_t *__restrict in, uint32_t *__restrict out);
159529
+ void __fastpack7(const uint64_t *__restrict in, uint32_t *__restrict out);
159530
+ void __fastpack8(const uint64_t *__restrict in, uint32_t *__restrict out);
159531
+ void __fastpack9(const uint64_t *__restrict in, uint32_t *__restrict out);
159532
+ void __fastpack10(const uint64_t *__restrict in, uint32_t *__restrict out);
159533
+ void __fastpack11(const uint64_t *__restrict in, uint32_t *__restrict out);
159534
+ void __fastpack12(const uint64_t *__restrict in, uint32_t *__restrict out);
159535
+ void __fastpack13(const uint64_t *__restrict in, uint32_t *__restrict out);
159536
+ void __fastpack14(const uint64_t *__restrict in, uint32_t *__restrict out);
159537
+ void __fastpack15(const uint64_t *__restrict in, uint32_t *__restrict out);
159538
+ void __fastpack16(const uint64_t *__restrict in, uint32_t *__restrict out);
159539
+ void __fastpack17(const uint64_t *__restrict in, uint32_t *__restrict out);
159540
+ void __fastpack18(const uint64_t *__restrict in, uint32_t *__restrict out);
159541
+ void __fastpack19(const uint64_t *__restrict in, uint32_t *__restrict out);
159542
+ void __fastpack20(const uint64_t *__restrict in, uint32_t *__restrict out);
159543
+ void __fastpack21(const uint64_t *__restrict in, uint32_t *__restrict out);
159544
+ void __fastpack22(const uint64_t *__restrict in, uint32_t *__restrict out);
159545
+ void __fastpack23(const uint64_t *__restrict in, uint32_t *__restrict out);
159546
+ void __fastpack24(const uint64_t *__restrict in, uint32_t *__restrict out);
159547
+ void __fastpack25(const uint64_t *__restrict in, uint32_t *__restrict out);
159548
+ void __fastpack26(const uint64_t *__restrict in, uint32_t *__restrict out);
159549
+ void __fastpack27(const uint64_t *__restrict in, uint32_t *__restrict out);
159550
+ void __fastpack28(const uint64_t *__restrict in, uint32_t *__restrict out);
159551
+ void __fastpack29(const uint64_t *__restrict in, uint32_t *__restrict out);
159552
+ void __fastpack30(const uint64_t *__restrict in, uint32_t *__restrict out);
159553
+ void __fastpack31(const uint64_t *__restrict in, uint32_t *__restrict out);
159554
+ void __fastpack32(const uint64_t *__restrict in, uint32_t *__restrict out);
159555
+ void __fastpack33(const uint64_t *__restrict in, uint32_t *__restrict out);
159556
+ void __fastpack34(const uint64_t *__restrict in, uint32_t *__restrict out);
159557
+ void __fastpack35(const uint64_t *__restrict in, uint32_t *__restrict out);
159558
+ void __fastpack36(const uint64_t *__restrict in, uint32_t *__restrict out);
159559
+ void __fastpack37(const uint64_t *__restrict in, uint32_t *__restrict out);
159560
+ void __fastpack38(const uint64_t *__restrict in, uint32_t *__restrict out);
159561
+ void __fastpack39(const uint64_t *__restrict in, uint32_t *__restrict out);
159562
+ void __fastpack40(const uint64_t *__restrict in, uint32_t *__restrict out);
159563
+ void __fastpack41(const uint64_t *__restrict in, uint32_t *__restrict out);
159564
+ void __fastpack42(const uint64_t *__restrict in, uint32_t *__restrict out);
159565
+ void __fastpack43(const uint64_t *__restrict in, uint32_t *__restrict out);
159566
+ void __fastpack44(const uint64_t *__restrict in, uint32_t *__restrict out);
159567
+ void __fastpack45(const uint64_t *__restrict in, uint32_t *__restrict out);
159568
+ void __fastpack46(const uint64_t *__restrict in, uint32_t *__restrict out);
159569
+ void __fastpack47(const uint64_t *__restrict in, uint32_t *__restrict out);
159570
+ void __fastpack48(const uint64_t *__restrict in, uint32_t *__restrict out);
159571
+ void __fastpack49(const uint64_t *__restrict in, uint32_t *__restrict out);
159572
+ void __fastpack50(const uint64_t *__restrict in, uint32_t *__restrict out);
159573
+ void __fastpack51(const uint64_t *__restrict in, uint32_t *__restrict out);
159574
+ void __fastpack52(const uint64_t *__restrict in, uint32_t *__restrict out);
159575
+ void __fastpack53(const uint64_t *__restrict in, uint32_t *__restrict out);
159576
+ void __fastpack54(const uint64_t *__restrict in, uint32_t *__restrict out);
159577
+ void __fastpack55(const uint64_t *__restrict in, uint32_t *__restrict out);
159578
+ void __fastpack56(const uint64_t *__restrict in, uint32_t *__restrict out);
159579
+ void __fastpack57(const uint64_t *__restrict in, uint32_t *__restrict out);
159580
+ void __fastpack58(const uint64_t *__restrict in, uint32_t *__restrict out);
159581
+ void __fastpack59(const uint64_t *__restrict in, uint32_t *__restrict out);
159582
+ void __fastpack60(const uint64_t *__restrict in, uint32_t *__restrict out);
159583
+ void __fastpack61(const uint64_t *__restrict in, uint32_t *__restrict out);
159584
+ void __fastpack62(const uint64_t *__restrict in, uint32_t *__restrict out);
159585
+ void __fastpack63(const uint64_t *__restrict in, uint32_t *__restrict out);
159586
+ void __fastpack64(const uint64_t *__restrict in, uint32_t *__restrict out);
159587
+ } // namespace internal
159588
+ } // namespace duckdb_fastpforlib
159589
+
159590
+
159591
+ // LICENSE_CHANGE_END
159592
+
159593
+
159594
+ #include <stdexcept>
159595
+
159596
+ namespace duckdb_fastpforlib {
159597
+
159598
+ namespace internal {
159599
+
159600
+ // Note that this only packs 8 values
159601
+ inline void fastunpack_quarter(const uint8_t *__restrict in, uint8_t *__restrict out, const uint32_t bit) {
159602
+ // Could have used function pointers instead of switch.
159603
+ // Switch calls do offer the compiler more opportunities for optimization in
159604
+ // theory. In this case, it makes no difference with a good compiler.
159605
+ switch (bit) {
159606
+ case 0:
159607
+ internal::__fastunpack0(in, out);
159608
+ break;
159609
+ case 1:
159610
+ internal::__fastunpack1(in, out);
159611
+ break;
159612
+ case 2:
159613
+ internal::__fastunpack2(in, out);
159614
+ break;
159615
+ case 3:
159616
+ internal::__fastunpack3(in, out);
159617
+ break;
159618
+ case 4:
159619
+ internal::__fastunpack4(in, out);
159620
+ break;
159621
+ case 5:
159622
+ internal::__fastunpack5(in, out);
159623
+ break;
159624
+ case 6:
159625
+ internal::__fastunpack6(in, out);
159626
+ break;
159627
+ case 7:
159628
+ internal::__fastunpack7(in, out);
159629
+ break;
159630
+ case 8:
159631
+ internal::__fastunpack8(in, out);
159632
+ break;
159633
+ default:
159634
+ throw std::logic_error("Invalid bit width for bitpacking");
159306
159635
  }
159307
- };
159636
+ }
159308
159637
 
159309
- struct FixedSizeUncompressed {
159310
- static CompressionFunction GetFunction(PhysicalType data_type);
159311
- };
159638
+ // Note that this only packs 8 values
159639
+ inline void fastpack_quarter(const uint8_t *__restrict in, uint8_t *__restrict out, const uint32_t bit) {
159640
+ // Could have used function pointers instead of switch.
159641
+ // Switch calls do offer the compiler more opportunities for optimization in
159642
+ // theory. In this case, it makes no difference with a good compiler.
159643
+ switch (bit) {
159644
+ case 0:
159645
+ internal::__fastpack0(in, out);
159646
+ break;
159647
+ case 1:
159648
+ internal::__fastpack1(in, out);
159649
+ break;
159650
+ case 2:
159651
+ internal::__fastpack2(in, out);
159652
+ break;
159653
+ case 3:
159654
+ internal::__fastpack3(in, out);
159655
+ break;
159656
+ case 4:
159657
+ internal::__fastpack4(in, out);
159658
+ break;
159659
+ case 5:
159660
+ internal::__fastpack5(in, out);
159661
+ break;
159662
+ case 6:
159663
+ internal::__fastpack6(in, out);
159664
+ break;
159665
+ case 7:
159666
+ internal::__fastpack7(in, out);
159667
+ break;
159668
+ case 8:
159669
+ internal::__fastpack8(in, out);
159670
+ break;
159671
+ default:
159672
+ throw std::logic_error("Invalid bit width for bitpacking");
159673
+ }
159674
+ }
159312
159675
 
159313
- struct ValidityUncompressed {
159314
- public:
159315
- static CompressionFunction GetFunction(PhysicalType data_type);
159676
+ // Note that this only packs 16 values
159677
+ inline void fastunpack_half(const uint16_t *__restrict in, uint16_t *__restrict out, const uint32_t bit) {
159678
+ // Could have used function pointers instead of switch.
159679
+ // Switch calls do offer the compiler more opportunities for optimization in
159680
+ // theory. In this case, it makes no difference with a good compiler.
159681
+ switch (bit) {
159682
+ case 0:
159683
+ internal::__fastunpack0(in, out);
159684
+ break;
159685
+ case 1:
159686
+ internal::__fastunpack1(in, out);
159687
+ break;
159688
+ case 2:
159689
+ internal::__fastunpack2(in, out);
159690
+ break;
159691
+ case 3:
159692
+ internal::__fastunpack3(in, out);
159693
+ break;
159694
+ case 4:
159695
+ internal::__fastunpack4(in, out);
159696
+ break;
159697
+ case 5:
159698
+ internal::__fastunpack5(in, out);
159699
+ break;
159700
+ case 6:
159701
+ internal::__fastunpack6(in, out);
159702
+ break;
159703
+ case 7:
159704
+ internal::__fastunpack7(in, out);
159705
+ break;
159706
+ case 8:
159707
+ internal::__fastunpack8(in, out);
159708
+ break;
159709
+ case 9:
159710
+ internal::__fastunpack9(in, out);
159711
+ break;
159712
+ case 10:
159713
+ internal::__fastunpack10(in, out);
159714
+ break;
159715
+ case 11:
159716
+ internal::__fastunpack11(in, out);
159717
+ break;
159718
+ case 12:
159719
+ internal::__fastunpack12(in, out);
159720
+ break;
159721
+ case 13:
159722
+ internal::__fastunpack13(in, out);
159723
+ break;
159724
+ case 14:
159725
+ internal::__fastunpack14(in, out);
159726
+ break;
159727
+ case 15:
159728
+ internal::__fastunpack15(in, out);
159729
+ break;
159730
+ case 16:
159731
+ internal::__fastunpack16(in, out);
159732
+ break;
159733
+ default:
159734
+ throw std::logic_error("Invalid bit width for bitpacking");
159735
+ }
159736
+ }
159316
159737
 
159317
- public:
159318
- static const validity_t LOWER_MASKS[65];
159319
- static const validity_t UPPER_MASKS[65];
159320
- };
159738
+ // Note that this only packs 16 values
159739
+ inline void fastpack_half(const uint16_t *__restrict in, uint16_t *__restrict out, const uint32_t bit) {
159740
+ // Could have used function pointers instead of switch.
159741
+ // Switch calls do offer the compiler more opportunities for optimization in
159742
+ // theory. In this case, it makes no difference with a good compiler.
159743
+ switch (bit) {
159744
+ case 0:
159745
+ internal::__fastpack0(in, out);
159746
+ break;
159747
+ case 1:
159748
+ internal::__fastpack1(in, out);
159749
+ break;
159750
+ case 2:
159751
+ internal::__fastpack2(in, out);
159752
+ break;
159753
+ case 3:
159754
+ internal::__fastpack3(in, out);
159755
+ break;
159756
+ case 4:
159757
+ internal::__fastpack4(in, out);
159758
+ break;
159759
+ case 5:
159760
+ internal::__fastpack5(in, out);
159761
+ break;
159762
+ case 6:
159763
+ internal::__fastpack6(in, out);
159764
+ break;
159765
+ case 7:
159766
+ internal::__fastpack7(in, out);
159767
+ break;
159768
+ case 8:
159769
+ internal::__fastpack8(in, out);
159770
+ break;
159771
+ case 9:
159772
+ internal::__fastpack9(in, out);
159773
+ break;
159774
+ case 10:
159775
+ internal::__fastpack10(in, out);
159776
+ break;
159777
+ case 11:
159778
+ internal::__fastpack11(in, out);
159779
+ break;
159780
+ case 12:
159781
+ internal::__fastpack12(in, out);
159782
+ break;
159783
+ case 13:
159784
+ internal::__fastpack13(in, out);
159785
+ break;
159786
+ case 14:
159787
+ internal::__fastpack14(in, out);
159788
+ break;
159789
+ case 15:
159790
+ internal::__fastpack15(in, out);
159791
+ break;
159792
+ case 16:
159793
+ internal::__fastpack16(in, out);
159794
+ break;
159795
+ default:
159796
+ throw std::logic_error("Invalid bit width for bitpacking");
159797
+ }
159798
+ }
159799
+ }
159321
159800
 
159322
- struct StringUncompressed {
159323
- public:
159324
- static CompressionFunction GetFunction(PhysicalType data_type);
159801
+ inline void fastunpack(const uint8_t *__restrict in, uint8_t *__restrict out, const uint32_t bit) {
159802
+ for (uint8_t i = 0; i < 4; i++) {
159803
+ internal::fastunpack_quarter(in + (i*bit), out+(i*8), bit);
159804
+ }
159805
+ }
159325
159806
 
159326
- public:
159327
- //! The max string size that is allowed within a block. Strings bigger than this will be labeled as a BIG STRING and
159328
- //! offloaded to the overflow blocks.
159329
- static constexpr uint16_t STRING_BLOCK_LIMIT = 4096;
159330
- };
159807
+ inline void fastunpack(const uint16_t *__restrict in, uint16_t *__restrict out, const uint32_t bit) {
159808
+ internal::fastunpack_half(in, out, bit);
159809
+ internal::fastunpack_half(in + bit, out+16, bit);
159810
+ }
159331
159811
 
159332
- } // namespace duckdb
159812
+ inline void fastunpack(const uint32_t *__restrict in,
159813
+ uint32_t *__restrict out, const uint32_t bit) {
159814
+ // Could have used function pointers instead of switch.
159815
+ // Switch calls do offer the compiler more opportunities for optimization in
159816
+ // theory. In this case, it makes no difference with a good compiler.
159817
+ switch (bit) {
159818
+ case 0:
159819
+ internal::__fastunpack0(in, out);
159820
+ break;
159821
+ case 1:
159822
+ internal::__fastunpack1(in, out);
159823
+ break;
159824
+ case 2:
159825
+ internal::__fastunpack2(in, out);
159826
+ break;
159827
+ case 3:
159828
+ internal::__fastunpack3(in, out);
159829
+ break;
159830
+ case 4:
159831
+ internal::__fastunpack4(in, out);
159832
+ break;
159833
+ case 5:
159834
+ internal::__fastunpack5(in, out);
159835
+ break;
159836
+ case 6:
159837
+ internal::__fastunpack6(in, out);
159838
+ break;
159839
+ case 7:
159840
+ internal::__fastunpack7(in, out);
159841
+ break;
159842
+ case 8:
159843
+ internal::__fastunpack8(in, out);
159844
+ break;
159845
+ case 9:
159846
+ internal::__fastunpack9(in, out);
159847
+ break;
159848
+ case 10:
159849
+ internal::__fastunpack10(in, out);
159850
+ break;
159851
+ case 11:
159852
+ internal::__fastunpack11(in, out);
159853
+ break;
159854
+ case 12:
159855
+ internal::__fastunpack12(in, out);
159856
+ break;
159857
+ case 13:
159858
+ internal::__fastunpack13(in, out);
159859
+ break;
159860
+ case 14:
159861
+ internal::__fastunpack14(in, out);
159862
+ break;
159863
+ case 15:
159864
+ internal::__fastunpack15(in, out);
159865
+ break;
159866
+ case 16:
159867
+ internal::__fastunpack16(in, out);
159868
+ break;
159869
+ case 17:
159870
+ internal::__fastunpack17(in, out);
159871
+ break;
159872
+ case 18:
159873
+ internal::__fastunpack18(in, out);
159874
+ break;
159875
+ case 19:
159876
+ internal::__fastunpack19(in, out);
159877
+ break;
159878
+ case 20:
159879
+ internal::__fastunpack20(in, out);
159880
+ break;
159881
+ case 21:
159882
+ internal::__fastunpack21(in, out);
159883
+ break;
159884
+ case 22:
159885
+ internal::__fastunpack22(in, out);
159886
+ break;
159887
+ case 23:
159888
+ internal::__fastunpack23(in, out);
159889
+ break;
159890
+ case 24:
159891
+ internal::__fastunpack24(in, out);
159892
+ break;
159893
+ case 25:
159894
+ internal::__fastunpack25(in, out);
159895
+ break;
159896
+ case 26:
159897
+ internal::__fastunpack26(in, out);
159898
+ break;
159899
+ case 27:
159900
+ internal::__fastunpack27(in, out);
159901
+ break;
159902
+ case 28:
159903
+ internal::__fastunpack28(in, out);
159904
+ break;
159905
+ case 29:
159906
+ internal::__fastunpack29(in, out);
159907
+ break;
159908
+ case 30:
159909
+ internal::__fastunpack30(in, out);
159910
+ break;
159911
+ case 31:
159912
+ internal::__fastunpack31(in, out);
159913
+ break;
159914
+ case 32:
159915
+ internal::__fastunpack32(in, out);
159916
+ break;
159917
+ default:
159918
+ throw std::logic_error("Invalid bit width for bitpacking");
159919
+ }
159920
+ }
159921
+
159922
+ inline void fastunpack(const uint32_t *__restrict in,
159923
+ uint64_t *__restrict out, const uint32_t bit) {
159924
+ // Could have used function pointers instead of switch.
159925
+ // Switch calls do offer the compiler more opportunities for optimization in
159926
+ // theory. In this case, it makes no difference with a good compiler.
159927
+ switch (bit) {
159928
+ case 0:
159929
+ internal::__fastunpack0(in, out);
159930
+ break;
159931
+ case 1:
159932
+ internal::__fastunpack1(in, out);
159933
+ break;
159934
+ case 2:
159935
+ internal::__fastunpack2(in, out);
159936
+ break;
159937
+ case 3:
159938
+ internal::__fastunpack3(in, out);
159939
+ break;
159940
+ case 4:
159941
+ internal::__fastunpack4(in, out);
159942
+ break;
159943
+ case 5:
159944
+ internal::__fastunpack5(in, out);
159945
+ break;
159946
+ case 6:
159947
+ internal::__fastunpack6(in, out);
159948
+ break;
159949
+ case 7:
159950
+ internal::__fastunpack7(in, out);
159951
+ break;
159952
+ case 8:
159953
+ internal::__fastunpack8(in, out);
159954
+ break;
159955
+ case 9:
159956
+ internal::__fastunpack9(in, out);
159957
+ break;
159958
+ case 10:
159959
+ internal::__fastunpack10(in, out);
159960
+ break;
159961
+ case 11:
159962
+ internal::__fastunpack11(in, out);
159963
+ break;
159964
+ case 12:
159965
+ internal::__fastunpack12(in, out);
159966
+ break;
159967
+ case 13:
159968
+ internal::__fastunpack13(in, out);
159969
+ break;
159970
+ case 14:
159971
+ internal::__fastunpack14(in, out);
159972
+ break;
159973
+ case 15:
159974
+ internal::__fastunpack15(in, out);
159975
+ break;
159976
+ case 16:
159977
+ internal::__fastunpack16(in, out);
159978
+ break;
159979
+ case 17:
159980
+ internal::__fastunpack17(in, out);
159981
+ break;
159982
+ case 18:
159983
+ internal::__fastunpack18(in, out);
159984
+ break;
159985
+ case 19:
159986
+ internal::__fastunpack19(in, out);
159987
+ break;
159988
+ case 20:
159989
+ internal::__fastunpack20(in, out);
159990
+ break;
159991
+ case 21:
159992
+ internal::__fastunpack21(in, out);
159993
+ break;
159994
+ case 22:
159995
+ internal::__fastunpack22(in, out);
159996
+ break;
159997
+ case 23:
159998
+ internal::__fastunpack23(in, out);
159999
+ break;
160000
+ case 24:
160001
+ internal::__fastunpack24(in, out);
160002
+ break;
160003
+ case 25:
160004
+ internal::__fastunpack25(in, out);
160005
+ break;
160006
+ case 26:
160007
+ internal::__fastunpack26(in, out);
160008
+ break;
160009
+ case 27:
160010
+ internal::__fastunpack27(in, out);
160011
+ break;
160012
+ case 28:
160013
+ internal::__fastunpack28(in, out);
160014
+ break;
160015
+ case 29:
160016
+ internal::__fastunpack29(in, out);
160017
+ break;
160018
+ case 30:
160019
+ internal::__fastunpack30(in, out);
160020
+ break;
160021
+ case 31:
160022
+ internal::__fastunpack31(in, out);
160023
+ break;
160024
+ case 32:
160025
+ internal::__fastunpack32(in, out);
160026
+ break;
160027
+ case 33:
160028
+ internal::__fastunpack33(in, out);
160029
+ break;
160030
+ case 34:
160031
+ internal::__fastunpack34(in, out);
160032
+ break;
160033
+ case 35:
160034
+ internal::__fastunpack35(in, out);
160035
+ break;
160036
+ case 36:
160037
+ internal::__fastunpack36(in, out);
160038
+ break;
160039
+ case 37:
160040
+ internal::__fastunpack37(in, out);
160041
+ break;
160042
+ case 38:
160043
+ internal::__fastunpack38(in, out);
160044
+ break;
160045
+ case 39:
160046
+ internal::__fastunpack39(in, out);
160047
+ break;
160048
+ case 40:
160049
+ internal::__fastunpack40(in, out);
160050
+ break;
160051
+ case 41:
160052
+ internal::__fastunpack41(in, out);
160053
+ break;
160054
+ case 42:
160055
+ internal::__fastunpack42(in, out);
160056
+ break;
160057
+ case 43:
160058
+ internal::__fastunpack43(in, out);
160059
+ break;
160060
+ case 44:
160061
+ internal::__fastunpack44(in, out);
160062
+ break;
160063
+ case 45:
160064
+ internal::__fastunpack45(in, out);
160065
+ break;
160066
+ case 46:
160067
+ internal::__fastunpack46(in, out);
160068
+ break;
160069
+ case 47:
160070
+ internal::__fastunpack47(in, out);
160071
+ break;
160072
+ case 48:
160073
+ internal::__fastunpack48(in, out);
160074
+ break;
160075
+ case 49:
160076
+ internal::__fastunpack49(in, out);
160077
+ break;
160078
+ case 50:
160079
+ internal::__fastunpack50(in, out);
160080
+ break;
160081
+ case 51:
160082
+ internal::__fastunpack51(in, out);
160083
+ break;
160084
+ case 52:
160085
+ internal::__fastunpack52(in, out);
160086
+ break;
160087
+ case 53:
160088
+ internal::__fastunpack53(in, out);
160089
+ break;
160090
+ case 54:
160091
+ internal::__fastunpack54(in, out);
160092
+ break;
160093
+ case 55:
160094
+ internal::__fastunpack55(in, out);
160095
+ break;
160096
+ case 56:
160097
+ internal::__fastunpack56(in, out);
160098
+ break;
160099
+ case 57:
160100
+ internal::__fastunpack57(in, out);
160101
+ break;
160102
+ case 58:
160103
+ internal::__fastunpack58(in, out);
160104
+ break;
160105
+ case 59:
160106
+ internal::__fastunpack59(in, out);
160107
+ break;
160108
+ case 60:
160109
+ internal::__fastunpack60(in, out);
160110
+ break;
160111
+ case 61:
160112
+ internal::__fastunpack61(in, out);
160113
+ break;
160114
+ case 62:
160115
+ internal::__fastunpack62(in, out);
160116
+ break;
160117
+ case 63:
160118
+ internal::__fastunpack63(in, out);
160119
+ break;
160120
+ case 64:
160121
+ internal::__fastunpack64(in, out);
160122
+ break;
160123
+ default:
160124
+ throw std::logic_error("Invalid bit width for bitpacking");
160125
+ }
160126
+ }
160127
+
160128
+ inline void fastpack(const uint8_t *__restrict in, uint8_t *__restrict out, const uint32_t bit) {
160129
+
160130
+ for (uint8_t i = 0; i < 4; i++) {
160131
+ internal::fastpack_quarter(in+(i*8), out + (i*bit), bit);
160132
+ }
160133
+ }
160134
+
160135
+ inline void fastpack(const uint16_t *__restrict in, uint16_t *__restrict out, const uint32_t bit) {
160136
+ internal::fastpack_half(in, out, bit);
160137
+ internal::fastpack_half(in+16, out + bit, bit);
160138
+ }
160139
+
160140
+ inline void fastpack(const uint32_t *__restrict in,
160141
+ uint32_t *__restrict out, const uint32_t bit) {
160142
+ // Could have used function pointers instead of switch.
160143
+ // Switch calls do offer the compiler more opportunities for optimization in
160144
+ // theory. In this case, it makes no difference with a good compiler.
160145
+ switch (bit) {
160146
+ case 0:
160147
+ internal::__fastpack0(in, out);
160148
+ break;
160149
+ case 1:
160150
+ internal::__fastpack1(in, out);
160151
+ break;
160152
+ case 2:
160153
+ internal::__fastpack2(in, out);
160154
+ break;
160155
+ case 3:
160156
+ internal::__fastpack3(in, out);
160157
+ break;
160158
+ case 4:
160159
+ internal::__fastpack4(in, out);
160160
+ break;
160161
+ case 5:
160162
+ internal::__fastpack5(in, out);
160163
+ break;
160164
+ case 6:
160165
+ internal::__fastpack6(in, out);
160166
+ break;
160167
+ case 7:
160168
+ internal::__fastpack7(in, out);
160169
+ break;
160170
+ case 8:
160171
+ internal::__fastpack8(in, out);
160172
+ break;
160173
+ case 9:
160174
+ internal::__fastpack9(in, out);
160175
+ break;
160176
+ case 10:
160177
+ internal::__fastpack10(in, out);
160178
+ break;
160179
+ case 11:
160180
+ internal::__fastpack11(in, out);
160181
+ break;
160182
+ case 12:
160183
+ internal::__fastpack12(in, out);
160184
+ break;
160185
+ case 13:
160186
+ internal::__fastpack13(in, out);
160187
+ break;
160188
+ case 14:
160189
+ internal::__fastpack14(in, out);
160190
+ break;
160191
+ case 15:
160192
+ internal::__fastpack15(in, out);
160193
+ break;
160194
+ case 16:
160195
+ internal::__fastpack16(in, out);
160196
+ break;
160197
+ case 17:
160198
+ internal::__fastpack17(in, out);
160199
+ break;
160200
+ case 18:
160201
+ internal::__fastpack18(in, out);
160202
+ break;
160203
+ case 19:
160204
+ internal::__fastpack19(in, out);
160205
+ break;
160206
+ case 20:
160207
+ internal::__fastpack20(in, out);
160208
+ break;
160209
+ case 21:
160210
+ internal::__fastpack21(in, out);
160211
+ break;
160212
+ case 22:
160213
+ internal::__fastpack22(in, out);
160214
+ break;
160215
+ case 23:
160216
+ internal::__fastpack23(in, out);
160217
+ break;
160218
+ case 24:
160219
+ internal::__fastpack24(in, out);
160220
+ break;
160221
+ case 25:
160222
+ internal::__fastpack25(in, out);
160223
+ break;
160224
+ case 26:
160225
+ internal::__fastpack26(in, out);
160226
+ break;
160227
+ case 27:
160228
+ internal::__fastpack27(in, out);
160229
+ break;
160230
+ case 28:
160231
+ internal::__fastpack28(in, out);
160232
+ break;
160233
+ case 29:
160234
+ internal::__fastpack29(in, out);
160235
+ break;
160236
+ case 30:
160237
+ internal::__fastpack30(in, out);
160238
+ break;
160239
+ case 31:
160240
+ internal::__fastpack31(in, out);
160241
+ break;
160242
+ case 32:
160243
+ internal::__fastpack32(in, out);
160244
+ break;
160245
+ default:
160246
+ throw std::logic_error("Invalid bit width for bitpacking");
160247
+ }
160248
+ }
160249
+
160250
+ inline void fastpack(const uint64_t *__restrict in,
160251
+ uint32_t *__restrict out, const uint32_t bit) {
160252
+ switch (bit) {
160253
+ case 0:
160254
+ internal::__fastpack0(in, out);
160255
+ break;
160256
+ case 1:
160257
+ internal::__fastpack1(in, out);
160258
+ break;
160259
+ case 2:
160260
+ internal::__fastpack2(in, out);
160261
+ break;
160262
+ case 3:
160263
+ internal::__fastpack3(in, out);
160264
+ break;
160265
+ case 4:
160266
+ internal::__fastpack4(in, out);
160267
+ break;
160268
+ case 5:
160269
+ internal::__fastpack5(in, out);
160270
+ break;
160271
+ case 6:
160272
+ internal::__fastpack6(in, out);
160273
+ break;
160274
+ case 7:
160275
+ internal::__fastpack7(in, out);
160276
+ break;
160277
+ case 8:
160278
+ internal::__fastpack8(in, out);
160279
+ break;
160280
+ case 9:
160281
+ internal::__fastpack9(in, out);
160282
+ break;
160283
+ case 10:
160284
+ internal::__fastpack10(in, out);
160285
+ break;
160286
+ case 11:
160287
+ internal::__fastpack11(in, out);
160288
+ break;
160289
+ case 12:
160290
+ internal::__fastpack12(in, out);
160291
+ break;
160292
+ case 13:
160293
+ internal::__fastpack13(in, out);
160294
+ break;
160295
+ case 14:
160296
+ internal::__fastpack14(in, out);
160297
+ break;
160298
+ case 15:
160299
+ internal::__fastpack15(in, out);
160300
+ break;
160301
+ case 16:
160302
+ internal::__fastpack16(in, out);
160303
+ break;
160304
+ case 17:
160305
+ internal::__fastpack17(in, out);
160306
+ break;
160307
+ case 18:
160308
+ internal::__fastpack18(in, out);
160309
+ break;
160310
+ case 19:
160311
+ internal::__fastpack19(in, out);
160312
+ break;
160313
+ case 20:
160314
+ internal::__fastpack20(in, out);
160315
+ break;
160316
+ case 21:
160317
+ internal::__fastpack21(in, out);
160318
+ break;
160319
+ case 22:
160320
+ internal::__fastpack22(in, out);
160321
+ break;
160322
+ case 23:
160323
+ internal::__fastpack23(in, out);
160324
+ break;
160325
+ case 24:
160326
+ internal::__fastpack24(in, out);
160327
+ break;
160328
+ case 25:
160329
+ internal::__fastpack25(in, out);
160330
+ break;
160331
+ case 26:
160332
+ internal::__fastpack26(in, out);
160333
+ break;
160334
+ case 27:
160335
+ internal::__fastpack27(in, out);
160336
+ break;
160337
+ case 28:
160338
+ internal::__fastpack28(in, out);
160339
+ break;
160340
+ case 29:
160341
+ internal::__fastpack29(in, out);
160342
+ break;
160343
+ case 30:
160344
+ internal::__fastpack30(in, out);
160345
+ break;
160346
+ case 31:
160347
+ internal::__fastpack31(in, out);
160348
+ break;
160349
+ case 32:
160350
+ internal::__fastpack32(in, out);
160351
+ break;
160352
+ case 33:
160353
+ internal::__fastpack33(in, out);
160354
+ break;
160355
+ case 34:
160356
+ internal::__fastpack34(in, out);
160357
+ break;
160358
+ case 35:
160359
+ internal::__fastpack35(in, out);
160360
+ break;
160361
+ case 36:
160362
+ internal::__fastpack36(in, out);
160363
+ break;
160364
+ case 37:
160365
+ internal::__fastpack37(in, out);
160366
+ break;
160367
+ case 38:
160368
+ internal::__fastpack38(in, out);
160369
+ break;
160370
+ case 39:
160371
+ internal::__fastpack39(in, out);
160372
+ break;
160373
+ case 40:
160374
+ internal::__fastpack40(in, out);
160375
+ break;
160376
+ case 41:
160377
+ internal::__fastpack41(in, out);
160378
+ break;
160379
+ case 42:
160380
+ internal::__fastpack42(in, out);
160381
+ break;
160382
+ case 43:
160383
+ internal::__fastpack43(in, out);
160384
+ break;
160385
+ case 44:
160386
+ internal::__fastpack44(in, out);
160387
+ break;
160388
+ case 45:
160389
+ internal::__fastpack45(in, out);
160390
+ break;
160391
+ case 46:
160392
+ internal::__fastpack46(in, out);
160393
+ break;
160394
+ case 47:
160395
+ internal::__fastpack47(in, out);
160396
+ break;
160397
+ case 48:
160398
+ internal::__fastpack48(in, out);
160399
+ break;
160400
+ case 49:
160401
+ internal::__fastpack49(in, out);
160402
+ break;
160403
+ case 50:
160404
+ internal::__fastpack50(in, out);
160405
+ break;
160406
+ case 51:
160407
+ internal::__fastpack51(in, out);
160408
+ break;
160409
+ case 52:
160410
+ internal::__fastpack52(in, out);
160411
+ break;
160412
+ case 53:
160413
+ internal::__fastpack53(in, out);
160414
+ break;
160415
+ case 54:
160416
+ internal::__fastpack54(in, out);
160417
+ break;
160418
+ case 55:
160419
+ internal::__fastpack55(in, out);
160420
+ break;
160421
+ case 56:
160422
+ internal::__fastpack56(in, out);
160423
+ break;
160424
+ case 57:
160425
+ internal::__fastpack57(in, out);
160426
+ break;
160427
+ case 58:
160428
+ internal::__fastpack58(in, out);
160429
+ break;
160430
+ case 59:
160431
+ internal::__fastpack59(in, out);
160432
+ break;
160433
+ case 60:
160434
+ internal::__fastpack60(in, out);
160435
+ break;
160436
+ case 61:
160437
+ internal::__fastpack61(in, out);
160438
+ break;
160439
+ case 62:
160440
+ internal::__fastpack62(in, out);
160441
+ break;
160442
+ case 63:
160443
+ internal::__fastpack63(in, out);
160444
+ break;
160445
+ case 64:
160446
+ internal::__fastpack64(in, out);
160447
+ break;
160448
+ default:
160449
+ throw std::logic_error("Invalid bit width for bitpacking");
160450
+ }
160451
+ }
160452
+ } // namespace fastpfor_lib
159333
160453
 
159334
160454
 
160455
+ // LICENSE_CHANGE_END
159335
160456
 
159336
160457
 
159337
160458
 
@@ -159578,6 +160699,691 @@ private:
159578
160699
  } // namespace duckdb
159579
160700
 
159580
160701
 
160702
+
160703
+ #include <functional>
160704
+
160705
+ namespace duckdb {
160706
+
160707
+ using bitpacking_width_t = uint8_t;
160708
+
160709
+ // Note that optimizations in scanning only work if this value is equal to STANDARD_VECTOR_SIZE, however we keep them
160710
+ // separated to prevent the code from break on lower vector sizes
160711
+ static constexpr const idx_t BITPACKING_WIDTH_GROUP_SIZE = 1024;
160712
+
160713
+ class BitpackingPrimitives {
160714
+ public:
160715
+ static constexpr const idx_t BITPACKING_ALGORITHM_GROUP_SIZE = 32;
160716
+ static constexpr const idx_t BITPACKING_HEADER_SIZE = sizeof(uint64_t);
160717
+ static constexpr const bool BYTE_ALIGNED = false;
160718
+
160719
+ // Packs a block of BITPACKING_ALGORITHM_GROUP_SIZE values
160720
+ template <class T>
160721
+ inline static void PackBlock(data_ptr_t dst, T *src, bitpacking_width_t width) {
160722
+ return PackGroup<T>(dst, src, width);
160723
+ }
160724
+
160725
+ // Unpacks a block of BITPACKING_ALGORITHM_GROUP_SIZE values
160726
+ template <class T>
160727
+ inline static void UnPackBlock(data_ptr_t dst, data_ptr_t src, bitpacking_width_t width,
160728
+ bool skip_sign_extension = false) {
160729
+ return UnPackGroup<T>(dst, src, width, skip_sign_extension);
160730
+ }
160731
+
160732
+ // Calculates the minimum required number of bits per value that can store all values
160733
+ template <class T>
160734
+ inline static bitpacking_width_t MinimumBitWidth(T *values, idx_t count) {
160735
+ return FindMinimumBitWidth<T, BYTE_ALIGNED>(values, count);
160736
+ }
160737
+
160738
+ private:
160739
+ template <class T>
160740
+ static bitpacking_width_t MinimumBitWidth(T min_value, T max_value) {
160741
+ bitpacking_width_t required_bits;
160742
+
160743
+ if (std::is_signed<T>::value) {
160744
+ if (min_value == NumericLimits<T>::Minimum()) {
160745
+ // handle special case of the minimal value, as it cannot be negated like all other values.
160746
+ return sizeof(T) * 8;
160747
+ } else {
160748
+ max_value = MaxValue((T)-min_value, max_value);
160749
+ }
160750
+ }
160751
+
160752
+ if (max_value == 0) {
160753
+ return 0;
160754
+ }
160755
+
160756
+ if (std::is_signed<T>::value) {
160757
+ required_bits = 1;
160758
+ } else {
160759
+ required_bits = 0;
160760
+ }
160761
+
160762
+ while (max_value) {
160763
+ required_bits++;
160764
+ max_value >>= 1;
160765
+ }
160766
+
160767
+ return GetEffectiveWidth<T>(required_bits);
160768
+ }
160769
+
160770
+ template <class T, bool round_to_next_byte = false>
160771
+ static bitpacking_width_t FindMinimumBitWidth(T *values, idx_t count) {
160772
+ T min_value = values[0];
160773
+ T max_value = values[0];
160774
+
160775
+ for (idx_t i = 1; i < count; i++) {
160776
+ if (values[i] > max_value) {
160777
+ max_value = values[i];
160778
+ }
160779
+
160780
+ if (std::is_signed<T>::value) {
160781
+ if (values[i] < min_value) {
160782
+ min_value = values[i];
160783
+ }
160784
+ }
160785
+ }
160786
+
160787
+ bitpacking_width_t calc_width = MinimumBitWidth<T>(std::is_signed<T>::value ? min_value : 0, max_value);
160788
+
160789
+ // Assert results are correct
160790
+ #ifdef DEBUG
160791
+ if (calc_width < sizeof(T) * 8 && calc_width != 0) {
160792
+ if (std::is_signed<T>::value) {
160793
+ D_ASSERT((int64_t)max_value <= (int64_t)(1L << (calc_width - 1)) - 1);
160794
+ D_ASSERT((int64_t)min_value >= (int64_t)(-1 * ((1L << (calc_width - 1)) - 1) - 1));
160795
+ } else {
160796
+ D_ASSERT((uint64_t)max_value <= (uint64_t)(1L << (calc_width)) - 1);
160797
+ }
160798
+ }
160799
+ #endif
160800
+ if (round_to_next_byte) {
160801
+ return (calc_width / 8 + (calc_width % 8 != 0)) * 8;
160802
+ } else {
160803
+ return calc_width;
160804
+ }
160805
+ }
160806
+
160807
+ template <class T>
160808
+ static void UnPackGroup(data_ptr_t dst, data_ptr_t src, bitpacking_width_t width,
160809
+ bool skip_sign_extension = false) {
160810
+ if (std::is_same<T, uint8_t>::value || std::is_same<T, int8_t>::value) {
160811
+ duckdb_fastpforlib::fastunpack((const uint8_t *)src, (uint8_t *)dst, (uint32_t)width);
160812
+ } else if (std::is_same<T, uint16_t>::value || std::is_same<T, int16_t>::value) {
160813
+ duckdb_fastpforlib::fastunpack((const uint16_t *)src, (uint16_t *)dst, (uint32_t)width);
160814
+ } else if (std::is_same<T, uint32_t>::value || std::is_same<T, int32_t>::value) {
160815
+ duckdb_fastpforlib::fastunpack((const uint32_t *)src, (uint32_t *)dst, (uint32_t)width);
160816
+ } else if (std::is_same<T, uint64_t>::value || std::is_same<T, int64_t>::value) {
160817
+ duckdb_fastpforlib::fastunpack((const uint32_t *)src, (uint64_t *)dst, (uint32_t)width);
160818
+ } else {
160819
+ throw InternalException("Unsupported type found in bitpacking.");
160820
+ }
160821
+
160822
+ if (NumericLimits<T>::IsSigned() && !skip_sign_extension && width > 0 && width < sizeof(T) * 8) {
160823
+ SignExtend<T>(dst, width);
160824
+ }
160825
+ }
160826
+
160827
+ // Prevent compression at widths that are ineffective
160828
+ template <class T>
160829
+ static bitpacking_width_t GetEffectiveWidth(bitpacking_width_t width) {
160830
+ if (width > 56) {
160831
+ return 64;
160832
+ }
160833
+
160834
+ if (width > 28 && (std::is_same<T, uint32_t>::value || std::is_same<T, int32_t>::value)) {
160835
+ return 32;
160836
+ }
160837
+
160838
+ else if (width > 14 && (std::is_same<T, uint16_t>::value || std::is_same<T, int16_t>::value)) {
160839
+ return 16;
160840
+ }
160841
+
160842
+ return width;
160843
+ }
160844
+
160845
+ // Sign bit extension
160846
+ template <class T, class T_U = typename std::make_unsigned<T>::type>
160847
+ static void SignExtend(data_ptr_t dst, bitpacking_width_t width) {
160848
+ T const mask = ((T_U)1) << (width - 1);
160849
+ for (idx_t i = 0; i < BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE; ++i) {
160850
+ T value = Load<T>(dst + i * sizeof(T));
160851
+ value = value & ((((T_U)1) << width) - ((T_U)1));
160852
+ T result = (value ^ mask) - mask;
160853
+ Store(result, dst + i * sizeof(T));
160854
+ }
160855
+ }
160856
+
160857
+ template <class T>
160858
+ static void PackGroup(data_ptr_t dst, T *values, bitpacking_width_t width) {
160859
+ if (std::is_same<T, uint8_t>::value || std::is_same<T, int8_t>::value) {
160860
+ duckdb_fastpforlib::fastpack((const uint8_t *)values, (uint8_t *)dst, (uint32_t)width);
160861
+ } else if (std::is_same<T, uint16_t>::value || std::is_same<T, int16_t>::value) {
160862
+ duckdb_fastpforlib::fastpack((const uint16_t *)values, (uint16_t *)dst, (uint32_t)width);
160863
+ } else if (std::is_same<T, uint32_t>::value || std::is_same<T, int32_t>::value) {
160864
+ duckdb_fastpforlib::fastpack((const uint32_t *)values, (uint32_t *)dst, (uint32_t)width);
160865
+ } else if (std::is_same<T, uint64_t>::value || std::is_same<T, int64_t>::value) {
160866
+ duckdb_fastpforlib::fastpack((const uint64_t *)values, (uint32_t *)dst, (uint32_t)width);
160867
+ } else {
160868
+ throw InternalException("Unsupported type found in bitpacking.");
160869
+ }
160870
+ }
160871
+ };
160872
+
160873
+ struct EmptyBitpackingWriter {
160874
+ template <class T>
160875
+ static void Operation(T *values, bool *validity, bitpacking_width_t width, idx_t count, void *data_ptr) {
160876
+ }
160877
+ };
160878
+
160879
+ template <class T>
160880
+ struct BitpackingState {
160881
+ public:
160882
+ BitpackingState() : compression_buffer_idx(0), total_size(0), data_ptr(nullptr) {
160883
+ }
160884
+
160885
+ T compression_buffer[BITPACKING_WIDTH_GROUP_SIZE];
160886
+ bool compression_buffer_validity[BITPACKING_WIDTH_GROUP_SIZE];
160887
+ idx_t compression_buffer_idx;
160888
+ idx_t total_size;
160889
+ void *data_ptr;
160890
+
160891
+ public:
160892
+ template <class OP>
160893
+ void Flush() {
160894
+ bitpacking_width_t width = BitpackingPrimitives::MinimumBitWidth<T>(compression_buffer, compression_buffer_idx);
160895
+ OP::Operation(compression_buffer, compression_buffer_validity, width, compression_buffer_idx, data_ptr);
160896
+ total_size += (BITPACKING_WIDTH_GROUP_SIZE * width) / 8 + sizeof(bitpacking_width_t);
160897
+ compression_buffer_idx = 0;
160898
+ }
160899
+
160900
+ template <class OP = EmptyBitpackingWriter>
160901
+ void Update(T *data, ValidityMask &validity, idx_t idx) {
160902
+
160903
+ if (validity.RowIsValid(idx)) {
160904
+ compression_buffer_validity[compression_buffer_idx] = true;
160905
+ compression_buffer[compression_buffer_idx++] = data[idx];
160906
+ } else {
160907
+ // We write zero for easy bitwidth analysis of the compression buffer later
160908
+ compression_buffer_validity[compression_buffer_idx] = false;
160909
+ compression_buffer[compression_buffer_idx++] = 0;
160910
+ }
160911
+
160912
+ if (compression_buffer_idx == BITPACKING_WIDTH_GROUP_SIZE) {
160913
+ // Calculate bitpacking width;
160914
+ Flush<OP>();
160915
+ }
160916
+ }
160917
+ };
160918
+
160919
+ //===--------------------------------------------------------------------===//
160920
+ // Analyze
160921
+ //===--------------------------------------------------------------------===//
160922
+ template <class T>
160923
+ struct BitpackingAnalyzeState : public AnalyzeState {
160924
+ BitpackingState<T> state;
160925
+ };
160926
+
160927
+ template <class T>
160928
+ unique_ptr<AnalyzeState> BitpackingInitAnalyze(ColumnData &col_data, PhysicalType type) {
160929
+ return make_unique<BitpackingAnalyzeState<T>>();
160930
+ }
160931
+
160932
+ template <class T>
160933
+ bool BitpackingAnalyze(AnalyzeState &state, Vector &input, idx_t count) {
160934
+ auto &analyze_state = (BitpackingAnalyzeState<T> &)state;
160935
+ VectorData vdata;
160936
+ input.Orrify(count, vdata);
160937
+
160938
+ auto data = (T *)vdata.data;
160939
+ for (idx_t i = 0; i < count; i++) {
160940
+ auto idx = vdata.sel->get_index(i);
160941
+ analyze_state.state.template Update<EmptyBitpackingWriter>(data, vdata.validity, idx);
160942
+ }
160943
+
160944
+ return true;
160945
+ }
160946
+
160947
+ template <class T>
160948
+ idx_t BitpackingFinalAnalyze(AnalyzeState &state) {
160949
+ auto &bitpacking_state = (BitpackingAnalyzeState<T> &)state;
160950
+ bitpacking_state.state.template Flush<EmptyBitpackingWriter>();
160951
+ return bitpacking_state.state.total_size;
160952
+ }
160953
+
160954
+ //===--------------------------------------------------------------------===//
160955
+ // Compress
160956
+ //===--------------------------------------------------------------------===//
160957
+ template <class T>
160958
+ struct BitpackingCompressState : public CompressionState {
160959
+ public:
160960
+ explicit BitpackingCompressState(ColumnDataCheckpointer &checkpointer) : checkpointer(checkpointer) {
160961
+ auto &db = checkpointer.GetDatabase();
160962
+ auto &type = checkpointer.GetType();
160963
+ auto &config = DBConfig::GetConfig(db);
160964
+ function = config.GetCompressionFunction(CompressionType::COMPRESSION_BITPACKING, type.InternalType());
160965
+ CreateEmptySegment(checkpointer.GetRowGroup().start);
160966
+
160967
+ state.data_ptr = (void *)this;
160968
+ }
160969
+
160970
+ ColumnDataCheckpointer &checkpointer;
160971
+ CompressionFunction *function;
160972
+ unique_ptr<ColumnSegment> current_segment;
160973
+ unique_ptr<BufferHandle> handle;
160974
+
160975
+ // Ptr to next free spot in segment;
160976
+ data_ptr_t data_ptr;
160977
+ // Ptr to next free spot for storing bitwidths (growing downwards).
160978
+ data_ptr_t width_ptr;
160979
+
160980
+ BitpackingState<T> state;
160981
+
160982
+ public:
160983
+ struct BitpackingWriter {
160984
+ template <class VALUE_TYPE>
160985
+ static void Operation(VALUE_TYPE *values, bool *validity, bitpacking_width_t width, idx_t count,
160986
+ void *data_ptr) {
160987
+ auto state = (BitpackingCompressState<T> *)data_ptr;
160988
+
160989
+ if (state->RemainingSize() < (width * BITPACKING_WIDTH_GROUP_SIZE) / 8 + sizeof(bitpacking_width_t)) {
160990
+ // Segment is full
160991
+ auto row_start = state->current_segment->start + state->current_segment->count;
160992
+ state->FlushSegment();
160993
+ state->CreateEmptySegment(row_start);
160994
+ }
160995
+
160996
+ for (idx_t i = 0; i < count; i++) {
160997
+ if (validity[i]) {
160998
+ NumericStatistics::Update<T>(state->current_segment->stats, values[i]);
160999
+ }
161000
+ }
161001
+
161002
+ state->WriteValues(values, width, count);
161003
+ }
161004
+ };
161005
+
161006
+ // Space remaining between the width_ptr growing down and data ptr growing up
161007
+ idx_t RemainingSize() {
161008
+ return width_ptr - data_ptr;
161009
+ }
161010
+
161011
+ void CreateEmptySegment(idx_t row_start) {
161012
+ auto &db = checkpointer.GetDatabase();
161013
+ auto &type = checkpointer.GetType();
161014
+ auto compressed_segment = ColumnSegment::CreateTransientSegment(db, type, row_start);
161015
+ compressed_segment->function = function;
161016
+ current_segment = move(compressed_segment);
161017
+ auto &buffer_manager = BufferManager::GetBufferManager(db);
161018
+ handle = buffer_manager.Pin(current_segment->block);
161019
+
161020
+ data_ptr = handle->Ptr() + current_segment->GetBlockOffset() + BitpackingPrimitives::BITPACKING_HEADER_SIZE;
161021
+ width_ptr =
161022
+ handle->Ptr() + current_segment->GetBlockOffset() + Storage::BLOCK_SIZE - sizeof(bitpacking_width_t);
161023
+ }
161024
+
161025
+ void Append(VectorData &vdata, idx_t count) {
161026
+ // TODO Optimization: avoid use of compression buffer if we can compress straight to result vector
161027
+ auto data = (T *)vdata.data;
161028
+
161029
+ for (idx_t i = 0; i < count; i++) {
161030
+ auto idx = vdata.sel->get_index(i);
161031
+ state.template Update<BitpackingCompressState<T>::BitpackingWriter>(data, vdata.validity, idx);
161032
+ }
161033
+ }
161034
+
161035
+ void WriteValues(T *values, bitpacking_width_t width, idx_t count) {
161036
+ // TODO we can optimize this by stopping early if count < BITPACKING_WIDTH_GROUP_SIZE
161037
+ idx_t compress_loops = BITPACKING_WIDTH_GROUP_SIZE / BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE;
161038
+ for (idx_t i = 0; i < compress_loops; i++) {
161039
+ BitpackingPrimitives::PackBlock<T>(
161040
+ data_ptr, &values[i * BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE], width);
161041
+ data_ptr += (BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE * width) / 8;
161042
+ }
161043
+
161044
+ Store<bitpacking_width_t>(width, width_ptr);
161045
+ width_ptr -= sizeof(bitpacking_width_t);
161046
+
161047
+ current_segment->count += count;
161048
+ }
161049
+
161050
+ void FlushSegment() {
161051
+ auto &state = checkpointer.GetCheckpointState();
161052
+
161053
+ // Compact the segment by moving the widths next to the data.
161054
+ idx_t minimal_widths_offset = AlignValue(data_ptr - handle->node->buffer);
161055
+ idx_t widths_size = handle->node->buffer + Storage::BLOCK_SIZE - width_ptr - 1;
161056
+ idx_t total_segment_size = minimal_widths_offset + widths_size;
161057
+ memmove(handle->node->buffer + minimal_widths_offset, width_ptr + 1, widths_size);
161058
+
161059
+ // Store the offset of the first width (which is at the highest address).
161060
+ Store<idx_t>(minimal_widths_offset + widths_size - 1, handle->node->buffer);
161061
+ handle.reset();
161062
+
161063
+ state.FlushSegment(move(current_segment), total_segment_size);
161064
+ }
161065
+
161066
+ void Finalize() {
161067
+ state.template Flush<BitpackingCompressState<T>::BitpackingWriter>();
161068
+ FlushSegment();
161069
+ current_segment.reset();
161070
+ }
161071
+ };
161072
+
161073
+ template <class T>
161074
+ unique_ptr<CompressionState> BitpackingInitCompression(ColumnDataCheckpointer &checkpointer,
161075
+ unique_ptr<AnalyzeState> state) {
161076
+ return make_unique<BitpackingCompressState<T>>(checkpointer);
161077
+ }
161078
+
161079
+ template <class T>
161080
+ void BitpackingCompress(CompressionState &state_p, Vector &scan_vector, idx_t count) {
161081
+ auto &state = (BitpackingCompressState<T> &)state_p;
161082
+ VectorData vdata;
161083
+ scan_vector.Orrify(count, vdata);
161084
+ state.Append(vdata, count);
161085
+ }
161086
+
161087
+ template <class T>
161088
+ void BitpackingFinalizeCompress(CompressionState &state_p) {
161089
+ auto &state = (BitpackingCompressState<T> &)state_p;
161090
+ state.Finalize();
161091
+ }
161092
+
161093
+ //===--------------------------------------------------------------------===//
161094
+ // Scan
161095
+ //===--------------------------------------------------------------------===//
161096
+ template <class T>
161097
+ struct BitpackingScanState : public SegmentScanState {
161098
+ public:
161099
+ explicit BitpackingScanState(ColumnSegment &segment) {
161100
+ auto &buffer_manager = BufferManager::GetBufferManager(segment.db);
161101
+ handle = buffer_manager.Pin(segment.block);
161102
+
161103
+ current_width_group_ptr =
161104
+ handle->node->buffer + segment.GetBlockOffset() + BitpackingPrimitives::BITPACKING_HEADER_SIZE;
161105
+
161106
+ // load offset to bitpacking widths pointer
161107
+ auto bitpacking_widths_offset = Load<idx_t>(handle->node->buffer + segment.GetBlockOffset());
161108
+ bitpacking_width_ptr = handle->node->buffer + segment.GetBlockOffset() + bitpacking_widths_offset;
161109
+
161110
+ // load the bitwidth of the first vector
161111
+ LoadCurrentBitWidth();
161112
+ }
161113
+
161114
+ unique_ptr<BufferHandle> handle;
161115
+
161116
+ void (*decompress_function)(data_ptr_t, data_ptr_t, bitpacking_width_t, bool skip_sign_extension);
161117
+ T decompression_buffer[BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE];
161118
+
161119
+ idx_t position_in_group = 0;
161120
+ data_ptr_t current_width_group_ptr;
161121
+ data_ptr_t bitpacking_width_ptr;
161122
+ bitpacking_width_t current_width;
161123
+
161124
+ public:
161125
+ void LoadCurrentBitWidth() {
161126
+ D_ASSERT(bitpacking_width_ptr > handle->node->buffer &&
161127
+ bitpacking_width_ptr < handle->node->buffer + Storage::BLOCK_SIZE);
161128
+ current_width = Load<bitpacking_width_t>(bitpacking_width_ptr);
161129
+ LoadDecompressFunction();
161130
+ }
161131
+
161132
+ void Skip(ColumnSegment &segment, idx_t skip_count) {
161133
+ while (skip_count > 0) {
161134
+ if (position_in_group + skip_count < BITPACKING_WIDTH_GROUP_SIZE) {
161135
+ // We're not leaving this bitpacking group, we can perform all skips.
161136
+ position_in_group += skip_count;
161137
+ break;
161138
+ } else {
161139
+ // The skip crosses the current bitpacking group, we skip the remainder of this group.
161140
+ auto skipping = BITPACKING_WIDTH_GROUP_SIZE - position_in_group;
161141
+ position_in_group = 0;
161142
+ current_width_group_ptr += (current_width * BITPACKING_WIDTH_GROUP_SIZE) / 8;
161143
+
161144
+ // Update width pointer and load new width
161145
+ bitpacking_width_ptr -= sizeof(bitpacking_width_t);
161146
+ LoadCurrentBitWidth();
161147
+
161148
+ skip_count -= skipping;
161149
+ }
161150
+ }
161151
+ }
161152
+
161153
+ void LoadDecompressFunction() {
161154
+ decompress_function = &BitpackingPrimitives::UnPackBlock<T>;
161155
+ }
161156
+ };
161157
+
161158
+ template <class T>
161159
+ unique_ptr<SegmentScanState> BitpackingInitScan(ColumnSegment &segment) {
161160
+ auto result = make_unique<BitpackingScanState<T>>(segment);
161161
+ return move(result);
161162
+ }
161163
+
161164
+ //===--------------------------------------------------------------------===//
161165
+ // Scan base data
161166
+ //===--------------------------------------------------------------------===//
161167
+ template <class T>
161168
+ void BitpackingScanPartial(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result,
161169
+ idx_t result_offset) {
161170
+ auto &scan_state = (BitpackingScanState<T> &)*state.scan_state;
161171
+
161172
+ T *result_data = FlatVector::GetData<T>(result);
161173
+ result.SetVectorType(VectorType::FLAT_VECTOR);
161174
+
161175
+ // Fast path for when no compression was used, we can do a single memcopy
161176
+ if (STANDARD_VECTOR_SIZE == BITPACKING_WIDTH_GROUP_SIZE) {
161177
+ if (scan_state.current_width == sizeof(T) * 8 && scan_count <= BITPACKING_WIDTH_GROUP_SIZE &&
161178
+ scan_state.position_in_group == 0) {
161179
+
161180
+ memcpy(result_data + result_offset, scan_state.current_width_group_ptr, scan_count * sizeof(T));
161181
+ scan_state.current_width_group_ptr += scan_count * sizeof(T);
161182
+ scan_state.bitpacking_width_ptr -= sizeof(bitpacking_width_t);
161183
+ scan_state.LoadCurrentBitWidth();
161184
+ return;
161185
+ }
161186
+ }
161187
+
161188
+ // Determine if we can skip sign extension during compression
161189
+ auto &nstats = (NumericStatistics &)*segment.stats.statistics;
161190
+ bool skip_sign_extend = std::is_signed<T>::value && nstats.min >= 0;
161191
+
161192
+ idx_t scanned = 0;
161193
+
161194
+ while (scanned < scan_count) {
161195
+ // Exhausted this width group, move pointers to next group and load bitwidth for next group.
161196
+ if (scan_state.position_in_group >= BITPACKING_WIDTH_GROUP_SIZE) {
161197
+ scan_state.position_in_group = 0;
161198
+ scan_state.bitpacking_width_ptr -= sizeof(bitpacking_width_t);
161199
+ scan_state.current_width_group_ptr += (scan_state.current_width * BITPACKING_WIDTH_GROUP_SIZE) / 8;
161200
+ scan_state.LoadCurrentBitWidth();
161201
+ }
161202
+
161203
+ idx_t offset_in_compression_group =
161204
+ scan_state.position_in_group % BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE;
161205
+
161206
+ idx_t to_scan = MinValue<idx_t>(scan_count - scanned, BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE -
161207
+ offset_in_compression_group);
161208
+
161209
+ // Calculate start of compression algorithm group
161210
+ data_ptr_t current_position_ptr =
161211
+ scan_state.current_width_group_ptr + scan_state.position_in_group * scan_state.current_width / 8;
161212
+ data_ptr_t decompression_group_start_pointer =
161213
+ current_position_ptr - offset_in_compression_group * scan_state.current_width / 8;
161214
+
161215
+ T *current_result_ptr = result_data + result_offset + scanned;
161216
+
161217
+ if (to_scan == BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE && offset_in_compression_group == 0) {
161218
+ // Decompress directly into result vector
161219
+ scan_state.decompress_function((data_ptr_t)current_result_ptr, decompression_group_start_pointer,
161220
+ scan_state.current_width, skip_sign_extend);
161221
+ } else {
161222
+ // Decompress compression algorithm to buffer
161223
+ scan_state.decompress_function((data_ptr_t)scan_state.decompression_buffer,
161224
+ decompression_group_start_pointer, scan_state.current_width,
161225
+ skip_sign_extend);
161226
+
161227
+ memcpy(current_result_ptr, scan_state.decompression_buffer + offset_in_compression_group,
161228
+ to_scan * sizeof(T));
161229
+ }
161230
+
161231
+ scanned += to_scan;
161232
+ scan_state.position_in_group += to_scan;
161233
+ }
161234
+ }
161235
+
161236
+ template <class T>
161237
+ void BitpackingScan(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result) {
161238
+ BitpackingScanPartial<T>(segment, state, scan_count, result, 0);
161239
+ }
161240
+
161241
+ //===--------------------------------------------------------------------===//
161242
+ // Fetch
161243
+ //===--------------------------------------------------------------------===//
161244
+ template <class T>
161245
+ void BitpackingFetchRow(ColumnSegment &segment, ColumnFetchState &state, row_t row_id, Vector &result,
161246
+ idx_t result_idx) {
161247
+ BitpackingScanState<T> scan_state(segment);
161248
+ scan_state.Skip(segment, row_id);
161249
+ auto result_data = FlatVector::GetData<T>(result);
161250
+ T *current_result_ptr = result_data + result_idx;
161251
+
161252
+ // TODO clean up, is reused in partialscan
161253
+ idx_t offset_in_compression_group =
161254
+ scan_state.position_in_group % BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE;
161255
+
161256
+ data_ptr_t decompression_group_start_pointer =
161257
+ scan_state.current_width_group_ptr +
161258
+ (scan_state.position_in_group - offset_in_compression_group) * scan_state.current_width / 8;
161259
+
161260
+ auto &nstats = (NumericStatistics &)*segment.stats.statistics;
161261
+ bool skip_sign_extend = std::is_signed<T>::value && nstats.min >= 0;
161262
+
161263
+ scan_state.decompress_function((data_ptr_t)scan_state.decompression_buffer, decompression_group_start_pointer,
161264
+ scan_state.current_width, skip_sign_extend);
161265
+
161266
+ *current_result_ptr = *(T *)(scan_state.decompression_buffer + offset_in_compression_group);
161267
+ }
161268
+ template <class T>
161269
+ void BitpackingSkip(ColumnSegment &segment, ColumnScanState &state, idx_t skip_count) {
161270
+ auto &scan_state = (BitpackingScanState<T> &)*state.scan_state;
161271
+ scan_state.Skip(segment, skip_count);
161272
+ }
161273
+
161274
+ //===--------------------------------------------------------------------===//
161275
+ // Get Function
161276
+ //===--------------------------------------------------------------------===//
161277
+ template <class T>
161278
+ CompressionFunction GetBitpackingFunction(PhysicalType data_type) {
161279
+ return CompressionFunction(CompressionType::COMPRESSION_BITPACKING, data_type, BitpackingInitAnalyze<T>,
161280
+ BitpackingAnalyze<T>, BitpackingFinalAnalyze<T>, BitpackingInitCompression<T>,
161281
+ BitpackingCompress<T>, BitpackingFinalizeCompress<T>, BitpackingInitScan<T>,
161282
+ BitpackingScan<T>, BitpackingScanPartial<T>, BitpackingFetchRow<T>, BitpackingSkip<T>);
161283
+ }
161284
+
161285
+ CompressionFunction BitpackingFun::GetFunction(PhysicalType type) {
161286
+ switch (type) {
161287
+ case PhysicalType::INT8:
161288
+ return GetBitpackingFunction<int8_t>(type);
161289
+ case PhysicalType::INT16:
161290
+ return GetBitpackingFunction<int16_t>(type);
161291
+ case PhysicalType::INT32:
161292
+ return GetBitpackingFunction<int32_t>(type);
161293
+ case PhysicalType::INT64:
161294
+ return GetBitpackingFunction<int64_t>(type);
161295
+ case PhysicalType::BOOL:
161296
+ case PhysicalType::UINT8:
161297
+ return GetBitpackingFunction<uint8_t>(type);
161298
+ case PhysicalType::UINT16:
161299
+ return GetBitpackingFunction<uint16_t>(type);
161300
+ case PhysicalType::UINT32:
161301
+ return GetBitpackingFunction<uint32_t>(type);
161302
+ case PhysicalType::UINT64:
161303
+ return GetBitpackingFunction<uint64_t>(type);
161304
+ default:
161305
+ throw InternalException("Unsupported type for Bitpacking");
161306
+ }
161307
+ }
161308
+
161309
+ bool BitpackingFun::TypeIsSupported(PhysicalType type) {
161310
+ switch (type) {
161311
+ case PhysicalType::BOOL:
161312
+ case PhysicalType::INT8:
161313
+ case PhysicalType::INT16:
161314
+ case PhysicalType::INT32:
161315
+ case PhysicalType::INT64:
161316
+ case PhysicalType::UINT8:
161317
+ case PhysicalType::UINT16:
161318
+ case PhysicalType::UINT32:
161319
+ case PhysicalType::UINT64:
161320
+ return true;
161321
+ default:
161322
+ return false;
161323
+ }
161324
+ }
161325
+
161326
+ } // namespace duckdb
161327
+ //===----------------------------------------------------------------------===//
161328
+ // DuckDB
161329
+ //
161330
+ // duckdb/storage/segment/uncompressed.hpp
161331
+ //
161332
+ //
161333
+ //===----------------------------------------------------------------------===//
161334
+
161335
+
161336
+
161337
+
161338
+
161339
+ namespace duckdb {
161340
+ class DatabaseInstance;
161341
+
161342
+ struct UncompressedFunctions {
161343
+ static unique_ptr<CompressionState> InitCompression(ColumnDataCheckpointer &checkpointer,
161344
+ unique_ptr<AnalyzeState> state);
161345
+ static void Compress(CompressionState &state_p, Vector &data, idx_t count);
161346
+ static void FinalizeCompress(CompressionState &state_p);
161347
+ static void EmptySkip(ColumnSegment &segment, ColumnScanState &state, idx_t skip_count) {
161348
+ }
161349
+ };
161350
+
161351
+ struct FixedSizeUncompressed {
161352
+ static CompressionFunction GetFunction(PhysicalType data_type);
161353
+ };
161354
+
161355
+ struct ValidityUncompressed {
161356
+ public:
161357
+ static CompressionFunction GetFunction(PhysicalType data_type);
161358
+
161359
+ public:
161360
+ static const validity_t LOWER_MASKS[65];
161361
+ static const validity_t UPPER_MASKS[65];
161362
+ };
161363
+
161364
+ struct StringUncompressed {
161365
+ public:
161366
+ static CompressionFunction GetFunction(PhysicalType data_type);
161367
+
161368
+ public:
161369
+ //! The max string size that is allowed within a block. Strings bigger than this will be labeled as a BIG STRING and
161370
+ //! offloaded to the overflow blocks.
161371
+ static constexpr uint16_t STRING_BLOCK_LIMIT = 4096;
161372
+ };
161373
+
161374
+ } // namespace duckdb
161375
+
161376
+
161377
+
161378
+
161379
+
161380
+
161381
+
161382
+
161383
+
161384
+
161385
+
161386
+
159581
161387
  namespace duckdb {
159582
161388
 
159583
161389
  //===--------------------------------------------------------------------===//
@@ -207501,6 +209307,1299 @@ void sdfree(void *ptr) { free(ptr); }
207501
209307
  // LICENSE_CHANGE_END
207502
209308
 
207503
209309
 
209310
+ // LICENSE_CHANGE_BEGIN
209311
+ // The following code up to LICENSE_CHANGE_END is subject to THIRD PARTY LICENSE #10
209312
+ // See the end of this file for a list
209313
+
209314
+
209315
+
209316
+ #include <cstdint>
209317
+ #include <type_traits>
209318
+
209319
+ namespace duckdb_fastpforlib {
209320
+ namespace internal {
209321
+
209322
+ // Used for uint8_t, uint16_t and uint32_t
209323
+ template <uint8_t DELTA, uint8_t SHR, class TYPE, uint8_t TYPE_SIZE = sizeof(TYPE) * 8>
209324
+ typename std::enable_if<(DELTA + SHR) < TYPE_SIZE>::type unpack_single_out(const TYPE *__restrict in,
209325
+ TYPE *__restrict out) {
209326
+ *out = ((*in) >> SHR) % (1 << DELTA);
209327
+ }
209328
+
209329
+ // Used for uint8_t, uint16_t and uint32_t
209330
+ template <uint8_t DELTA, uint8_t SHR, class TYPE, uint8_t TYPE_SIZE = sizeof(TYPE) * 8>
209331
+ typename std::enable_if<(DELTA + SHR) >= TYPE_SIZE>::type unpack_single_out(const TYPE *__restrict &in,
209332
+ TYPE *__restrict out) {
209333
+ *out = (*in) >> SHR;
209334
+ ++in;
209335
+
209336
+ static const TYPE NEXT_SHR = SHR + DELTA - TYPE_SIZE;
209337
+ *out |= ((*in) % (1U << NEXT_SHR)) << (TYPE_SIZE - SHR);
209338
+ }
209339
+
209340
+ template <uint8_t DELTA, uint8_t SHR>
209341
+ typename std::enable_if<(DELTA + SHR) < 32>::type unpack_single_out(const uint32_t *__restrict in,
209342
+ uint64_t *__restrict out) {
209343
+ *out = ((static_cast<uint64_t>(*in)) >> SHR) % (1ULL << DELTA);
209344
+ }
209345
+
209346
+ template <uint8_t DELTA, uint8_t SHR>
209347
+ typename std::enable_if<(DELTA + SHR) >= 32 && (DELTA + SHR) < 64>::type
209348
+ unpack_single_out(const uint32_t *__restrict &in, uint64_t *__restrict out) {
209349
+ *out = static_cast<uint64_t>(*in) >> SHR;
209350
+ ++in;
209351
+ if (DELTA + SHR > 32) {
209352
+ static const uint8_t NEXT_SHR = SHR + DELTA - 32;
209353
+ *out |= static_cast<uint64_t>((*in) % (1U << NEXT_SHR)) << (32 - SHR);
209354
+ }
209355
+ }
209356
+
209357
+ template <uint8_t DELTA, uint8_t SHR>
209358
+ typename std::enable_if<(DELTA + SHR) >= 64>::type unpack_single_out(const uint32_t *__restrict &in,
209359
+ uint64_t *__restrict out) {
209360
+ *out = static_cast<uint64_t>(*in) >> SHR;
209361
+ ++in;
209362
+
209363
+ *out |= static_cast<uint64_t>(*in) << (32 - SHR);
209364
+ ++in;
209365
+
209366
+ if (DELTA + SHR > 64) {
209367
+ static const uint8_t NEXT_SHR = DELTA + SHR - 64;
209368
+ *out |= static_cast<uint64_t>((*in) % (1U << NEXT_SHR)) << (64 - SHR);
209369
+ }
209370
+ }
209371
+
209372
+ // Used for uint8_t, uint16_t and uint32_t
209373
+ template <class TYPE, uint16_t DELTA, uint16_t SHL, TYPE MASK, uint8_t TYPE_SIZE = sizeof(TYPE) * 8>
209374
+ typename std::enable_if < DELTA + SHL<TYPE_SIZE>::type pack_single_in(const TYPE in, TYPE *__restrict out) {
209375
+ if (SHL == 0) {
209376
+ *out = in & MASK;
209377
+ } else {
209378
+ *out |= (in & MASK) << SHL;
209379
+ }
209380
+ }
209381
+
209382
+ // Used for uint8_t, uint16_t and uint32_t
209383
+ template <class TYPE, uint16_t DELTA, uint16_t SHL, TYPE MASK, uint8_t TYPE_SIZE = sizeof(TYPE) * 8>
209384
+ typename std::enable_if<DELTA + SHL >= TYPE_SIZE>::type pack_single_in(const TYPE in, TYPE *__restrict &out) {
209385
+ *out |= in << SHL;
209386
+ ++out;
209387
+
209388
+ if (DELTA + SHL > TYPE_SIZE) {
209389
+ *out = (in & MASK) >> (TYPE_SIZE - SHL);
209390
+ }
209391
+ }
209392
+
209393
+ template <uint16_t DELTA, uint16_t SHL, uint64_t MASK>
209394
+ typename std::enable_if < DELTA + SHL<32>::type pack_single_in64(const uint64_t in, uint32_t *__restrict out) {
209395
+ if (SHL == 0) {
209396
+ *out = static_cast<uint32_t>(in & MASK);
209397
+ } else {
209398
+ *out |= (in & MASK) << SHL;
209399
+ }
209400
+ }
209401
+ template <uint16_t DELTA, uint16_t SHL, uint64_t MASK>
209402
+ typename std::enable_if < DELTA + SHL >= 32 &&
209403
+ DELTA + SHL<64>::type pack_single_in64(const uint64_t in, uint32_t *__restrict &out) {
209404
+ if (SHL == 0) {
209405
+ *out = static_cast<uint32_t>(in & MASK);
209406
+ } else {
209407
+ *out |= (in & MASK) << SHL;
209408
+ }
209409
+
209410
+ ++out;
209411
+
209412
+ if (DELTA + SHL > 32) {
209413
+ *out = static_cast<uint32_t>((in & MASK) >> (32 - SHL));
209414
+ }
209415
+ }
209416
+ template <uint16_t DELTA, uint16_t SHL, uint64_t MASK>
209417
+ typename std::enable_if<DELTA + SHL >= 64>::type pack_single_in64(const uint64_t in, uint32_t *__restrict &out) {
209418
+ *out |= in << SHL;
209419
+ ++out;
209420
+
209421
+ *out = static_cast<uint32_t>((in & MASK) >> (32 - SHL));
209422
+ ++out;
209423
+
209424
+ if (DELTA + SHL > 64) {
209425
+ *out = (in & MASK) >> (64 - SHL);
209426
+ }
209427
+ }
209428
+ template <uint16_t DELTA, uint16_t OINDEX = 0>
209429
+ struct Unroller8 {
209430
+ static void Unpack(const uint8_t *__restrict &in, uint8_t *__restrict out) {
209431
+ unpack_single_out<DELTA, (DELTA * OINDEX) % 8>(in, out + OINDEX);
209432
+
209433
+ Unroller8<DELTA, OINDEX + 1>::Unpack(in, out);
209434
+ }
209435
+
209436
+ static void Pack(const uint8_t *__restrict in, uint8_t *__restrict out) {
209437
+ pack_single_in<uint8_t, DELTA, (DELTA * OINDEX) % 8, (1U << DELTA) - 1>(in[OINDEX], out);
209438
+
209439
+ Unroller8<DELTA, OINDEX + 1>::Pack(in, out);
209440
+ }
209441
+
209442
+ };\
209443
+ template <uint16_t DELTA>
209444
+ struct Unroller8<DELTA, 7> {
209445
+ enum { SHIFT = (DELTA * 7) % 8 };
209446
+
209447
+ static void Unpack(const uint8_t *__restrict in, uint8_t *__restrict out) {
209448
+ out[7] = (*in) >> SHIFT;
209449
+ }
209450
+
209451
+ static void Pack(const uint8_t *__restrict in, uint8_t *__restrict out) {
209452
+ *out |= (in[7] << SHIFT);
209453
+ }
209454
+ };
209455
+
209456
+ template <uint16_t DELTA, uint16_t OINDEX = 0>
209457
+ struct Unroller16 {
209458
+ static void Unpack(const uint16_t *__restrict &in, uint16_t *__restrict out) {
209459
+ unpack_single_out<DELTA, (DELTA * OINDEX) % 16>(in, out + OINDEX);
209460
+
209461
+ Unroller16<DELTA, OINDEX + 1>::Unpack(in, out);
209462
+ }
209463
+
209464
+ static void Pack(const uint16_t *__restrict in, uint16_t *__restrict out) {
209465
+ pack_single_in<uint16_t, DELTA, (DELTA * OINDEX) % 16, (1U << DELTA) - 1>(in[OINDEX], out);
209466
+
209467
+ Unroller16<DELTA, OINDEX + 1>::Pack(in, out);
209468
+ }
209469
+
209470
+ };
209471
+
209472
+ template <uint16_t DELTA>
209473
+ struct Unroller16<DELTA, 15> {
209474
+ enum { SHIFT = (DELTA * 15) % 16 };
209475
+
209476
+ static void Unpack(const uint16_t *__restrict in, uint16_t *__restrict out) {
209477
+ out[15] = (*in) >> SHIFT;
209478
+ }
209479
+
209480
+ static void Pack(const uint16_t *__restrict in, uint16_t *__restrict out) {
209481
+ *out |= (in[15] << SHIFT);
209482
+ }
209483
+ };
209484
+
209485
+ template <uint16_t DELTA, uint16_t OINDEX = 0>
209486
+ struct Unroller {
209487
+ static void Unpack(const uint32_t *__restrict &in, uint32_t *__restrict out) {
209488
+ unpack_single_out<DELTA, (DELTA * OINDEX) % 32>(in, out + OINDEX);
209489
+
209490
+ Unroller<DELTA, OINDEX + 1>::Unpack(in, out);
209491
+ }
209492
+
209493
+ static void Unpack(const uint32_t *__restrict &in, uint64_t *__restrict out) {
209494
+ unpack_single_out<DELTA, (DELTA * OINDEX) % 32>(in, out + OINDEX);
209495
+
209496
+ Unroller<DELTA, OINDEX + 1>::Unpack(in, out);
209497
+ }
209498
+
209499
+ static void Pack(const uint32_t *__restrict in, uint32_t *__restrict out) {
209500
+ pack_single_in<uint32_t, DELTA, (DELTA * OINDEX) % 32, (1U << DELTA) - 1>(in[OINDEX], out);
209501
+
209502
+ Unroller<DELTA, OINDEX + 1>::Pack(in, out);
209503
+ }
209504
+
209505
+ static void Pack(const uint64_t *__restrict in, uint32_t *__restrict out) {
209506
+ pack_single_in64<DELTA, (DELTA * OINDEX) % 32, (1ULL << DELTA) - 1>(in[OINDEX], out);
209507
+
209508
+ Unroller<DELTA, OINDEX + 1>::Pack(in, out);
209509
+ }
209510
+ };
209511
+
209512
+ template <uint16_t DELTA>
209513
+ struct Unroller<DELTA, 31> {
209514
+ enum { SHIFT = (DELTA * 31) % 32 };
209515
+
209516
+ static void Unpack(const uint32_t *__restrict in, uint32_t *__restrict out) {
209517
+ out[31] = (*in) >> SHIFT;
209518
+ }
209519
+
209520
+ static void Unpack(const uint32_t *__restrict in, uint64_t *__restrict out) {
209521
+ out[31] = (*in) >> SHIFT;
209522
+ if (DELTA > 32) {
209523
+ ++in;
209524
+ out[31] |= static_cast<uint64_t>(*in) << (32 - SHIFT);
209525
+ }
209526
+ }
209527
+
209528
+ static void Pack(const uint32_t *__restrict in, uint32_t *__restrict out) {
209529
+ *out |= (in[31] << SHIFT);
209530
+ }
209531
+
209532
+ static void Pack(const uint64_t *__restrict in, uint32_t *__restrict out) {
209533
+ *out |= (in[31] << SHIFT);
209534
+ if (DELTA > 32) {
209535
+ ++out;
209536
+ *out = static_cast<uint32_t>(in[31] >> (32 - SHIFT));
209537
+ }
209538
+ }
209539
+ };
209540
+
209541
+ // Special cases
209542
+ void __fastunpack0(const uint8_t *__restrict, uint8_t *__restrict out) {
209543
+ for (uint8_t i = 0; i < 8; ++i)
209544
+ *(out++) = 0;
209545
+ }
209546
+
209547
+ void __fastunpack0(const uint16_t *__restrict, uint16_t *__restrict out) {
209548
+ for (uint16_t i = 0; i < 16; ++i)
209549
+ *(out++) = 0;
209550
+ }
209551
+
209552
+ void __fastunpack0(const uint32_t *__restrict, uint32_t *__restrict out) {
209553
+ for (uint32_t i = 0; i < 32; ++i)
209554
+ *(out++) = 0;
209555
+ }
209556
+
209557
+ void __fastunpack0(const uint32_t *__restrict, uint64_t *__restrict out) {
209558
+ for (uint32_t i = 0; i < 32; ++i)
209559
+ *(out++) = 0;
209560
+ }
209561
+
209562
+ void __fastpack0(const uint8_t *__restrict, uint8_t *__restrict) {
209563
+ }
209564
+ void __fastpack0(const uint16_t *__restrict, uint16_t *__restrict) {
209565
+ }
209566
+ void __fastpack0(const uint32_t *__restrict, uint32_t *__restrict) {
209567
+ }
209568
+ void __fastpack0(const uint64_t *__restrict, uint32_t *__restrict) {
209569
+ }
209570
+
209571
+ // fastunpack for 8 bits
209572
+ void __fastunpack1(const uint8_t *__restrict in, uint8_t *__restrict out) {
209573
+ Unroller8<1>::Unpack(in, out);
209574
+ }
209575
+
209576
+ void __fastunpack2(const uint8_t *__restrict in, uint8_t *__restrict out) {
209577
+ Unroller8<2>::Unpack(in, out);
209578
+ }
209579
+
209580
+ void __fastunpack3(const uint8_t *__restrict in, uint8_t *__restrict out) {
209581
+ Unroller8<3>::Unpack(in, out);
209582
+ }
209583
+
209584
+ void __fastunpack4(const uint8_t *__restrict in, uint8_t *__restrict out) {
209585
+ for (uint8_t outer = 0; outer < 4; ++outer) {
209586
+ for (uint8_t inwordpointer = 0; inwordpointer < 8; inwordpointer += 4)
209587
+ *(out++) = ((*in) >> inwordpointer) % (1U << 4);
209588
+ ++in;
209589
+ }
209590
+ }
209591
+
209592
+ void __fastunpack5(const uint8_t *__restrict in, uint8_t *__restrict out) {
209593
+ Unroller8<5>::Unpack(in, out);
209594
+ }
209595
+
209596
+ void __fastunpack6(const uint8_t *__restrict in, uint8_t *__restrict out) {
209597
+ Unroller8<6>::Unpack(in, out);
209598
+ }
209599
+
209600
+ void __fastunpack7(const uint8_t *__restrict in, uint8_t *__restrict out) {
209601
+ Unroller8<7>::Unpack(in, out);
209602
+ }
209603
+
209604
+ void __fastunpack8(const uint8_t *__restrict in, uint8_t *__restrict out) {
209605
+ for (int k = 0; k < 8; ++k)
209606
+ out[k] = in[k];
209607
+ }
209608
+
209609
+
209610
+ // fastunpack for 16 bits
209611
+ void __fastunpack1(const uint16_t *__restrict in, uint16_t *__restrict out) {
209612
+ Unroller16<1>::Unpack(in, out);
209613
+ }
209614
+
209615
+ void __fastunpack2(const uint16_t *__restrict in, uint16_t *__restrict out) {
209616
+ Unroller16<2>::Unpack(in, out);
209617
+ }
209618
+
209619
+ void __fastunpack3(const uint16_t *__restrict in, uint16_t *__restrict out) {
209620
+ Unroller16<3>::Unpack(in, out);
209621
+ }
209622
+
209623
+ void __fastunpack4(const uint16_t *__restrict in, uint16_t *__restrict out) {
209624
+ for (uint16_t outer = 0; outer < 4; ++outer) {
209625
+ for (uint16_t inwordpointer = 0; inwordpointer < 16; inwordpointer += 4)
209626
+ *(out++) = ((*in) >> inwordpointer) % (1U << 4);
209627
+ ++in;
209628
+ }
209629
+ }
209630
+
209631
+ void __fastunpack5(const uint16_t *__restrict in, uint16_t *__restrict out) {
209632
+ Unroller16<5>::Unpack(in, out);
209633
+ }
209634
+
209635
+ void __fastunpack6(const uint16_t *__restrict in, uint16_t *__restrict out) {
209636
+ Unroller16<6>::Unpack(in, out);
209637
+ }
209638
+
209639
+ void __fastunpack7(const uint16_t *__restrict in, uint16_t *__restrict out) {
209640
+ Unroller16<7>::Unpack(in, out);
209641
+ }
209642
+
209643
+ void __fastunpack8(const uint16_t *__restrict in, uint16_t *__restrict out) {
209644
+ for (uint16_t outer = 0; outer < 8; ++outer) {
209645
+ for (uint16_t inwordpointer = 0; inwordpointer < 16; inwordpointer += 8)
209646
+ *(out++) = ((*in) >> inwordpointer) % (1U << 8);
209647
+ ++in;
209648
+ }
209649
+ }
209650
+
209651
+ void __fastunpack9(const uint16_t *__restrict in, uint16_t *__restrict out) {
209652
+ Unroller16<9>::Unpack(in, out);
209653
+ }
209654
+
209655
+ void __fastunpack10(const uint16_t *__restrict in, uint16_t *__restrict out) {
209656
+ Unroller16<10>::Unpack(in, out);
209657
+ }
209658
+
209659
+ void __fastunpack11(const uint16_t *__restrict in, uint16_t *__restrict out) {
209660
+ Unroller16<11>::Unpack(in, out);
209661
+ }
209662
+
209663
+ void __fastunpack12(const uint16_t *__restrict in, uint16_t *__restrict out) {
209664
+ Unroller16<12>::Unpack(in, out);
209665
+ }
209666
+
209667
+ void __fastunpack13(const uint16_t *__restrict in, uint16_t *__restrict out) {
209668
+ Unroller16<13>::Unpack(in, out);
209669
+ }
209670
+
209671
+ void __fastunpack14(const uint16_t *__restrict in, uint16_t *__restrict out) {
209672
+ Unroller16<14>::Unpack(in, out);
209673
+ }
209674
+
209675
+ void __fastunpack15(const uint16_t *__restrict in, uint16_t *__restrict out) {
209676
+ Unroller16<15>::Unpack(in, out);
209677
+ }
209678
+
209679
+ void __fastunpack16(const uint16_t *__restrict in, uint16_t *__restrict out) {
209680
+ for (int k = 0; k < 16; ++k)
209681
+ out[k] = in[k];
209682
+ }
209683
+
209684
+ // fastunpack for 32 bits
209685
+ void __fastunpack1(const uint32_t *__restrict in, uint32_t *__restrict out) {
209686
+ Unroller<1>::Unpack(in, out);
209687
+ }
209688
+
209689
+ void __fastunpack2(const uint32_t *__restrict in, uint32_t *__restrict out) {
209690
+ Unroller<2>::Unpack(in, out);
209691
+ }
209692
+
209693
+ void __fastunpack3(const uint32_t *__restrict in, uint32_t *__restrict out) {
209694
+ Unroller<3>::Unpack(in, out);
209695
+ }
209696
+
209697
+ void __fastunpack4(const uint32_t *__restrict in, uint32_t *__restrict out) {
209698
+ for (uint32_t outer = 0; outer < 4; ++outer) {
209699
+ for (uint32_t inwordpointer = 0; inwordpointer < 32; inwordpointer += 4)
209700
+ *(out++) = ((*in) >> inwordpointer) % (1U << 4);
209701
+ ++in;
209702
+ }
209703
+ }
209704
+
209705
+ void __fastunpack5(const uint32_t *__restrict in, uint32_t *__restrict out) {
209706
+ Unroller<5>::Unpack(in, out);
209707
+ }
209708
+
209709
+ void __fastunpack6(const uint32_t *__restrict in, uint32_t *__restrict out) {
209710
+ Unroller<6>::Unpack(in, out);
209711
+ }
209712
+
209713
+ void __fastunpack7(const uint32_t *__restrict in, uint32_t *__restrict out) {
209714
+ Unroller<7>::Unpack(in, out);
209715
+ }
209716
+
209717
+ void __fastunpack8(const uint32_t *__restrict in, uint32_t *__restrict out) {
209718
+ for (uint32_t outer = 0; outer < 8; ++outer) {
209719
+ for (uint32_t inwordpointer = 0; inwordpointer < 32; inwordpointer += 8)
209720
+ *(out++) = ((*in) >> inwordpointer) % (1U << 8);
209721
+ ++in;
209722
+ }
209723
+ }
209724
+
209725
+ void __fastunpack9(const uint32_t *__restrict in, uint32_t *__restrict out) {
209726
+ Unroller<9>::Unpack(in, out);
209727
+ }
209728
+
209729
+ void __fastunpack10(const uint32_t *__restrict in, uint32_t *__restrict out) {
209730
+ Unroller<10>::Unpack(in, out);
209731
+ }
209732
+
209733
+ void __fastunpack11(const uint32_t *__restrict in, uint32_t *__restrict out) {
209734
+ Unroller<11>::Unpack(in, out);
209735
+ }
209736
+
209737
+ void __fastunpack12(const uint32_t *__restrict in, uint32_t *__restrict out) {
209738
+ Unroller<12>::Unpack(in, out);
209739
+ }
209740
+
209741
+ void __fastunpack13(const uint32_t *__restrict in, uint32_t *__restrict out) {
209742
+ Unroller<13>::Unpack(in, out);
209743
+ }
209744
+
209745
+ void __fastunpack14(const uint32_t *__restrict in, uint32_t *__restrict out) {
209746
+ Unroller<14>::Unpack(in, out);
209747
+ }
209748
+
209749
+ void __fastunpack15(const uint32_t *__restrict in, uint32_t *__restrict out) {
209750
+ Unroller<15>::Unpack(in, out);
209751
+ }
209752
+
209753
+ void __fastunpack16(const uint32_t *__restrict in, uint32_t *__restrict out) {
209754
+ for (uint32_t outer = 0; outer < 16; ++outer) {
209755
+ for (uint32_t inwordpointer = 0; inwordpointer < 32; inwordpointer += 16)
209756
+ *(out++) = ((*in) >> inwordpointer) % (1U << 16);
209757
+ ++in;
209758
+ }
209759
+ }
209760
+
209761
+ void __fastunpack17(const uint32_t *__restrict in, uint32_t *__restrict out) {
209762
+ Unroller<17>::Unpack(in, out);
209763
+ }
209764
+
209765
+ void __fastunpack18(const uint32_t *__restrict in, uint32_t *__restrict out) {
209766
+ Unroller<18>::Unpack(in, out);
209767
+ }
209768
+
209769
+ void __fastunpack19(const uint32_t *__restrict in, uint32_t *__restrict out) {
209770
+ Unroller<19>::Unpack(in, out);
209771
+ }
209772
+
209773
+ void __fastunpack20(const uint32_t *__restrict in, uint32_t *__restrict out) {
209774
+ Unroller<20>::Unpack(in, out);
209775
+ }
209776
+
209777
+ void __fastunpack21(const uint32_t *__restrict in, uint32_t *__restrict out) {
209778
+ Unroller<21>::Unpack(in, out);
209779
+ }
209780
+
209781
+ void __fastunpack22(const uint32_t *__restrict in, uint32_t *__restrict out) {
209782
+ Unroller<22>::Unpack(in, out);
209783
+ }
209784
+
209785
+ void __fastunpack23(const uint32_t *__restrict in, uint32_t *__restrict out) {
209786
+ Unroller<23>::Unpack(in, out);
209787
+ }
209788
+
209789
+ void __fastunpack24(const uint32_t *__restrict in, uint32_t *__restrict out) {
209790
+ Unroller<24>::Unpack(in, out);
209791
+ }
209792
+
209793
+ void __fastunpack25(const uint32_t *__restrict in, uint32_t *__restrict out) {
209794
+ Unroller<25>::Unpack(in, out);
209795
+ }
209796
+
209797
+ void __fastunpack26(const uint32_t *__restrict in, uint32_t *__restrict out) {
209798
+ Unroller<26>::Unpack(in, out);
209799
+ }
209800
+
209801
+ void __fastunpack27(const uint32_t *__restrict in, uint32_t *__restrict out) {
209802
+ Unroller<27>::Unpack(in, out);
209803
+ }
209804
+
209805
+ void __fastunpack28(const uint32_t *__restrict in, uint32_t *__restrict out) {
209806
+ Unroller<28>::Unpack(in, out);
209807
+ }
209808
+
209809
+ void __fastunpack29(const uint32_t *__restrict in, uint32_t *__restrict out) {
209810
+ Unroller<29>::Unpack(in, out);
209811
+ }
209812
+
209813
+ void __fastunpack30(const uint32_t *__restrict in, uint32_t *__restrict out) {
209814
+ Unroller<30>::Unpack(in, out);
209815
+ }
209816
+
209817
+ void __fastunpack31(const uint32_t *__restrict in, uint32_t *__restrict out) {
209818
+ Unroller<31>::Unpack(in, out);
209819
+ }
209820
+
209821
+ void __fastunpack32(const uint32_t *__restrict in, uint32_t *__restrict out) {
209822
+ for (int k = 0; k < 32; ++k)
209823
+ out[k] = in[k];
209824
+ }
209825
+
209826
+ // fastupack for 64 bits
209827
+ void __fastunpack1(const uint32_t *__restrict in, uint64_t *__restrict out) {
209828
+ Unroller<1>::Unpack(in, out);
209829
+ }
209830
+
209831
+ void __fastunpack2(const uint32_t *__restrict in, uint64_t *__restrict out) {
209832
+ Unroller<2>::Unpack(in, out);
209833
+ }
209834
+
209835
+ void __fastunpack3(const uint32_t *__restrict in, uint64_t *__restrict out) {
209836
+ Unroller<3>::Unpack(in, out);
209837
+ }
209838
+
209839
+ void __fastunpack4(const uint32_t *__restrict in, uint64_t *__restrict out) {
209840
+ for (uint32_t outer = 0; outer < 4; ++outer) {
209841
+ for (uint32_t inwordpointer = 0; inwordpointer < 32; inwordpointer += 4)
209842
+ *(out++) = ((*in) >> inwordpointer) % (1U << 4);
209843
+ ++in;
209844
+ }
209845
+ }
209846
+
209847
+ void __fastunpack5(const uint32_t *__restrict in, uint64_t *__restrict out) {
209848
+ Unroller<5>::Unpack(in, out);
209849
+ }
209850
+
209851
+ void __fastunpack6(const uint32_t *__restrict in, uint64_t *__restrict out) {
209852
+ Unroller<6>::Unpack(in, out);
209853
+ }
209854
+
209855
+ void __fastunpack7(const uint32_t *__restrict in, uint64_t *__restrict out) {
209856
+ Unroller<7>::Unpack(in, out);
209857
+ }
209858
+
209859
+ void __fastunpack8(const uint32_t *__restrict in, uint64_t *__restrict out) {
209860
+ for (uint32_t outer = 0; outer < 8; ++outer) {
209861
+ for (uint32_t inwordpointer = 0; inwordpointer < 32; inwordpointer += 8) {
209862
+ *(out++) = ((*in) >> inwordpointer) % (1U << 8);
209863
+ }
209864
+ ++in;
209865
+ }
209866
+ }
209867
+
209868
+ void __fastunpack9(const uint32_t *__restrict in, uint64_t *__restrict out) {
209869
+ Unroller<9>::Unpack(in, out);
209870
+ }
209871
+
209872
+ void __fastunpack10(const uint32_t *__restrict in, uint64_t *__restrict out) {
209873
+ Unroller<10>::Unpack(in, out);
209874
+ }
209875
+
209876
+ void __fastunpack11(const uint32_t *__restrict in, uint64_t *__restrict out) {
209877
+ Unroller<11>::Unpack(in, out);
209878
+ }
209879
+
209880
+ void __fastunpack12(const uint32_t *__restrict in, uint64_t *__restrict out) {
209881
+ Unroller<12>::Unpack(in, out);
209882
+ }
209883
+
209884
+ void __fastunpack13(const uint32_t *__restrict in, uint64_t *__restrict out) {
209885
+ Unroller<13>::Unpack(in, out);
209886
+ }
209887
+
209888
+ void __fastunpack14(const uint32_t *__restrict in, uint64_t *__restrict out) {
209889
+ Unroller<14>::Unpack(in, out);
209890
+ }
209891
+
209892
+ void __fastunpack15(const uint32_t *__restrict in, uint64_t *__restrict out) {
209893
+ Unroller<15>::Unpack(in, out);
209894
+ }
209895
+
209896
+ void __fastunpack16(const uint32_t *__restrict in, uint64_t *__restrict out) {
209897
+ for (uint32_t outer = 0; outer < 16; ++outer) {
209898
+ for (uint32_t inwordpointer = 0; inwordpointer < 32; inwordpointer += 16)
209899
+ *(out++) = ((*in) >> inwordpointer) % (1U << 16);
209900
+ ++in;
209901
+ }
209902
+ }
209903
+
209904
+ void __fastunpack17(const uint32_t *__restrict in, uint64_t *__restrict out) {
209905
+ Unroller<17>::Unpack(in, out);
209906
+ }
209907
+
209908
+ void __fastunpack18(const uint32_t *__restrict in, uint64_t *__restrict out) {
209909
+ Unroller<18>::Unpack(in, out);
209910
+ }
209911
+
209912
+ void __fastunpack19(const uint32_t *__restrict in, uint64_t *__restrict out) {
209913
+ Unroller<19>::Unpack(in, out);
209914
+ }
209915
+
209916
+ void __fastunpack20(const uint32_t *__restrict in, uint64_t *__restrict out) {
209917
+ Unroller<20>::Unpack(in, out);
209918
+ }
209919
+
209920
+ void __fastunpack21(const uint32_t *__restrict in, uint64_t *__restrict out) {
209921
+ Unroller<21>::Unpack(in, out);
209922
+ }
209923
+
209924
+ void __fastunpack22(const uint32_t *__restrict in, uint64_t *__restrict out) {
209925
+ Unroller<22>::Unpack(in, out);
209926
+ }
209927
+
209928
+ void __fastunpack23(const uint32_t *__restrict in, uint64_t *__restrict out) {
209929
+ Unroller<23>::Unpack(in, out);
209930
+ }
209931
+
209932
+ void __fastunpack24(const uint32_t *__restrict in, uint64_t *__restrict out) {
209933
+ Unroller<24>::Unpack(in, out);
209934
+ }
209935
+
209936
+ void __fastunpack25(const uint32_t *__restrict in, uint64_t *__restrict out) {
209937
+ Unroller<25>::Unpack(in, out);
209938
+ }
209939
+
209940
+ void __fastunpack26(const uint32_t *__restrict in, uint64_t *__restrict out) {
209941
+ Unroller<26>::Unpack(in, out);
209942
+ }
209943
+
209944
+ void __fastunpack27(const uint32_t *__restrict in, uint64_t *__restrict out) {
209945
+ Unroller<27>::Unpack(in, out);
209946
+ }
209947
+
209948
+ void __fastunpack28(const uint32_t *__restrict in, uint64_t *__restrict out) {
209949
+ Unroller<28>::Unpack(in, out);
209950
+ }
209951
+
209952
+ void __fastunpack29(const uint32_t *__restrict in, uint64_t *__restrict out) {
209953
+ Unroller<29>::Unpack(in, out);
209954
+ }
209955
+
209956
+ void __fastunpack30(const uint32_t *__restrict in, uint64_t *__restrict out) {
209957
+ Unroller<30>::Unpack(in, out);
209958
+ }
209959
+
209960
+ void __fastunpack31(const uint32_t *__restrict in, uint64_t *__restrict out) {
209961
+ Unroller<31>::Unpack(in, out);
209962
+ }
209963
+
209964
+ void __fastunpack32(const uint32_t *__restrict in, uint64_t *__restrict out) {
209965
+ for (int k = 0; k < 32; ++k)
209966
+ out[k] = in[k];
209967
+ }
209968
+
209969
+ void __fastunpack33(const uint32_t *__restrict in, uint64_t *__restrict out) {
209970
+ Unroller<33>::Unpack(in, out);
209971
+ }
209972
+
209973
+ void __fastunpack34(const uint32_t *__restrict in, uint64_t *__restrict out) {
209974
+ Unroller<34>::Unpack(in, out);
209975
+ }
209976
+
209977
+ void __fastunpack35(const uint32_t *__restrict in, uint64_t *__restrict out) {
209978
+ Unroller<35>::Unpack(in, out);
209979
+ }
209980
+
209981
+ void __fastunpack36(const uint32_t *__restrict in, uint64_t *__restrict out) {
209982
+ Unroller<36>::Unpack(in, out);
209983
+ }
209984
+
209985
+ void __fastunpack37(const uint32_t *__restrict in, uint64_t *__restrict out) {
209986
+ Unroller<37>::Unpack(in, out);
209987
+ }
209988
+
209989
+ void __fastunpack38(const uint32_t *__restrict in, uint64_t *__restrict out) {
209990
+ Unroller<38>::Unpack(in, out);
209991
+ }
209992
+
209993
+ void __fastunpack39(const uint32_t *__restrict in, uint64_t *__restrict out) {
209994
+ Unroller<39>::Unpack(in, out);
209995
+ }
209996
+
209997
+ void __fastunpack40(const uint32_t *__restrict in, uint64_t *__restrict out) {
209998
+ Unroller<40>::Unpack(in, out);
209999
+ }
210000
+
210001
+ void __fastunpack41(const uint32_t *__restrict in, uint64_t *__restrict out) {
210002
+ Unroller<41>::Unpack(in, out);
210003
+ }
210004
+
210005
+ void __fastunpack42(const uint32_t *__restrict in, uint64_t *__restrict out) {
210006
+ Unroller<42>::Unpack(in, out);
210007
+ }
210008
+
210009
+ void __fastunpack43(const uint32_t *__restrict in, uint64_t *__restrict out) {
210010
+ Unroller<43>::Unpack(in, out);
210011
+ }
210012
+
210013
+ void __fastunpack44(const uint32_t *__restrict in, uint64_t *__restrict out) {
210014
+ Unroller<44>::Unpack(in, out);
210015
+ }
210016
+
210017
+ void __fastunpack45(const uint32_t *__restrict in, uint64_t *__restrict out) {
210018
+ Unroller<45>::Unpack(in, out);
210019
+ }
210020
+
210021
+ void __fastunpack46(const uint32_t *__restrict in, uint64_t *__restrict out) {
210022
+ Unroller<46>::Unpack(in, out);
210023
+ }
210024
+
210025
+ void __fastunpack47(const uint32_t *__restrict in, uint64_t *__restrict out) {
210026
+ Unroller<47>::Unpack(in, out);
210027
+ }
210028
+
210029
+ void __fastunpack48(const uint32_t *__restrict in, uint64_t *__restrict out) {
210030
+ Unroller<48>::Unpack(in, out);
210031
+ }
210032
+
210033
+ void __fastunpack49(const uint32_t *__restrict in, uint64_t *__restrict out) {
210034
+ Unroller<49>::Unpack(in, out);
210035
+ }
210036
+
210037
+ void __fastunpack50(const uint32_t *__restrict in, uint64_t *__restrict out) {
210038
+ Unroller<50>::Unpack(in, out);
210039
+ }
210040
+
210041
+ void __fastunpack51(const uint32_t *__restrict in, uint64_t *__restrict out) {
210042
+ Unroller<51>::Unpack(in, out);
210043
+ }
210044
+
210045
+ void __fastunpack52(const uint32_t *__restrict in, uint64_t *__restrict out) {
210046
+ Unroller<52>::Unpack(in, out);
210047
+ }
210048
+
210049
+ void __fastunpack53(const uint32_t *__restrict in, uint64_t *__restrict out) {
210050
+ Unroller<53>::Unpack(in, out);
210051
+ }
210052
+
210053
+ void __fastunpack54(const uint32_t *__restrict in, uint64_t *__restrict out) {
210054
+ Unroller<54>::Unpack(in, out);
210055
+ }
210056
+
210057
+ void __fastunpack55(const uint32_t *__restrict in, uint64_t *__restrict out) {
210058
+ Unroller<55>::Unpack(in, out);
210059
+ }
210060
+
210061
+ void __fastunpack56(const uint32_t *__restrict in, uint64_t *__restrict out) {
210062
+ Unroller<56>::Unpack(in, out);
210063
+ }
210064
+
210065
+ void __fastunpack57(const uint32_t *__restrict in, uint64_t *__restrict out) {
210066
+ Unroller<57>::Unpack(in, out);
210067
+ }
210068
+
210069
+ void __fastunpack58(const uint32_t *__restrict in, uint64_t *__restrict out) {
210070
+ Unroller<58>::Unpack(in, out);
210071
+ }
210072
+
210073
+ void __fastunpack59(const uint32_t *__restrict in, uint64_t *__restrict out) {
210074
+ Unroller<59>::Unpack(in, out);
210075
+ }
210076
+
210077
+ void __fastunpack60(const uint32_t *__restrict in, uint64_t *__restrict out) {
210078
+ Unroller<60>::Unpack(in, out);
210079
+ }
210080
+
210081
+ void __fastunpack61(const uint32_t *__restrict in, uint64_t *__restrict out) {
210082
+ Unroller<61>::Unpack(in, out);
210083
+ }
210084
+
210085
+ void __fastunpack62(const uint32_t *__restrict in, uint64_t *__restrict out) {
210086
+ Unroller<62>::Unpack(in, out);
210087
+ }
210088
+
210089
+ void __fastunpack63(const uint32_t *__restrict in, uint64_t *__restrict out) {
210090
+ Unroller<63>::Unpack(in, out);
210091
+ }
210092
+
210093
+ void __fastunpack64(const uint32_t *__restrict in, uint64_t *__restrict out) {
210094
+ for (int k = 0; k < 32; ++k) {
210095
+ out[k] = in[k * 2];
210096
+ out[k] |= static_cast<uint64_t>(in[k * 2 + 1]) << 32;
210097
+ }
210098
+ }
210099
+
210100
+ // fastpack for 8 bits
210101
+
210102
+ void __fastpack1(const uint8_t *__restrict in, uint8_t *__restrict out) {
210103
+ Unroller8<1>::Pack(in, out);
210104
+ }
210105
+
210106
+ void __fastpack2(const uint8_t *__restrict in, uint8_t *__restrict out) {
210107
+ Unroller8<2>::Pack(in, out);
210108
+ }
210109
+
210110
+ void __fastpack3(const uint8_t *__restrict in, uint8_t *__restrict out) {
210111
+ Unroller8<3>::Pack(in, out);
210112
+ }
210113
+
210114
+ void __fastpack4(const uint8_t *__restrict in, uint8_t *__restrict out) {
210115
+ Unroller8<4>::Pack(in, out);
210116
+ }
210117
+
210118
+ void __fastpack5(const uint8_t *__restrict in, uint8_t *__restrict out) {
210119
+ Unroller8<5>::Pack(in, out);
210120
+ }
210121
+
210122
+ void __fastpack6(const uint8_t *__restrict in, uint8_t *__restrict out) {
210123
+ Unroller8<6>::Pack(in, out);
210124
+ }
210125
+
210126
+ void __fastpack7(const uint8_t *__restrict in, uint8_t *__restrict out) {
210127
+ Unroller8<7>::Pack(in, out);
210128
+ }
210129
+
210130
+ void __fastpack8(const uint8_t *__restrict in, uint8_t *__restrict out) {
210131
+ for (int k = 0; k < 8; ++k)
210132
+ out[k] = in[k];
210133
+ }
210134
+
210135
+ // fastpack for 16 bits
210136
+
210137
+ void __fastpack1(const uint16_t *__restrict in, uint16_t *__restrict out) {
210138
+ Unroller16<1>::Pack(in, out);
210139
+ }
210140
+
210141
+ void __fastpack2(const uint16_t *__restrict in, uint16_t *__restrict out) {
210142
+ Unroller16<2>::Pack(in, out);
210143
+ }
210144
+
210145
+ void __fastpack3(const uint16_t *__restrict in, uint16_t *__restrict out) {
210146
+ Unroller16<3>::Pack(in, out);
210147
+ }
210148
+
210149
+ void __fastpack4(const uint16_t *__restrict in, uint16_t *__restrict out) {
210150
+ Unroller16<4>::Pack(in, out);
210151
+ }
210152
+
210153
+ void __fastpack5(const uint16_t *__restrict in, uint16_t *__restrict out) {
210154
+ Unroller16<5>::Pack(in, out);
210155
+ }
210156
+
210157
+ void __fastpack6(const uint16_t *__restrict in, uint16_t *__restrict out) {
210158
+ Unroller16<6>::Pack(in, out);
210159
+ }
210160
+
210161
+ void __fastpack7(const uint16_t *__restrict in, uint16_t *__restrict out) {
210162
+ Unroller16<7>::Pack(in, out);
210163
+ }
210164
+
210165
+ void __fastpack8(const uint16_t *__restrict in, uint16_t *__restrict out) {
210166
+ Unroller16<8>::Pack(in, out);
210167
+ }
210168
+
210169
+ void __fastpack9(const uint16_t *__restrict in, uint16_t *__restrict out) {
210170
+ Unroller16<9>::Pack(in, out);
210171
+ }
210172
+
210173
+ void __fastpack10(const uint16_t *__restrict in, uint16_t *__restrict out) {
210174
+ Unroller16<10>::Pack(in, out);
210175
+ }
210176
+
210177
+ void __fastpack11(const uint16_t *__restrict in, uint16_t *__restrict out) {
210178
+ Unroller16<11>::Pack(in, out);
210179
+ }
210180
+
210181
+ void __fastpack12(const uint16_t *__restrict in, uint16_t *__restrict out) {
210182
+ Unroller16<12>::Pack(in, out);
210183
+ }
210184
+
210185
+ void __fastpack13(const uint16_t *__restrict in, uint16_t *__restrict out) {
210186
+ Unroller16<13>::Pack(in, out);
210187
+ }
210188
+
210189
+ void __fastpack14(const uint16_t *__restrict in, uint16_t *__restrict out) {
210190
+ Unroller16<14>::Pack(in, out);
210191
+ }
210192
+
210193
+ void __fastpack15(const uint16_t *__restrict in, uint16_t *__restrict out) {
210194
+ Unroller16<15>::Pack(in, out);
210195
+ }
210196
+
210197
+ void __fastpack16(const uint16_t *__restrict in, uint16_t *__restrict out) {
210198
+ for (int k = 0; k < 16; ++k)
210199
+ out[k] = in[k];
210200
+ }
210201
+
210202
+
210203
+ // fastpack for 32 bits
210204
+
210205
+ void __fastpack1(const uint32_t *__restrict in, uint32_t *__restrict out) {
210206
+ Unroller<1>::Pack(in, out);
210207
+ }
210208
+
210209
+ void __fastpack2(const uint32_t *__restrict in, uint32_t *__restrict out) {
210210
+ Unroller<2>::Pack(in, out);
210211
+ }
210212
+
210213
+ void __fastpack3(const uint32_t *__restrict in, uint32_t *__restrict out) {
210214
+ Unroller<3>::Pack(in, out);
210215
+ }
210216
+
210217
+ void __fastpack4(const uint32_t *__restrict in, uint32_t *__restrict out) {
210218
+ Unroller<4>::Pack(in, out);
210219
+ }
210220
+
210221
+ void __fastpack5(const uint32_t *__restrict in, uint32_t *__restrict out) {
210222
+ Unroller<5>::Pack(in, out);
210223
+ }
210224
+
210225
+ void __fastpack6(const uint32_t *__restrict in, uint32_t *__restrict out) {
210226
+ Unroller<6>::Pack(in, out);
210227
+ }
210228
+
210229
+ void __fastpack7(const uint32_t *__restrict in, uint32_t *__restrict out) {
210230
+ Unroller<7>::Pack(in, out);
210231
+ }
210232
+
210233
+ void __fastpack8(const uint32_t *__restrict in, uint32_t *__restrict out) {
210234
+ Unroller<8>::Pack(in, out);
210235
+ }
210236
+
210237
+ void __fastpack9(const uint32_t *__restrict in, uint32_t *__restrict out) {
210238
+ Unroller<9>::Pack(in, out);
210239
+ }
210240
+
210241
+ void __fastpack10(const uint32_t *__restrict in, uint32_t *__restrict out) {
210242
+ Unroller<10>::Pack(in, out);
210243
+ }
210244
+
210245
+ void __fastpack11(const uint32_t *__restrict in, uint32_t *__restrict out) {
210246
+ Unroller<11>::Pack(in, out);
210247
+ }
210248
+
210249
+ void __fastpack12(const uint32_t *__restrict in, uint32_t *__restrict out) {
210250
+ Unroller<12>::Pack(in, out);
210251
+ }
210252
+
210253
+ void __fastpack13(const uint32_t *__restrict in, uint32_t *__restrict out) {
210254
+ Unroller<13>::Pack(in, out);
210255
+ }
210256
+
210257
+ void __fastpack14(const uint32_t *__restrict in, uint32_t *__restrict out) {
210258
+ Unroller<14>::Pack(in, out);
210259
+ }
210260
+
210261
+ void __fastpack15(const uint32_t *__restrict in, uint32_t *__restrict out) {
210262
+ Unroller<15>::Pack(in, out);
210263
+ }
210264
+
210265
+ void __fastpack16(const uint32_t *__restrict in, uint32_t *__restrict out) {
210266
+ Unroller<16>::Pack(in, out);
210267
+ }
210268
+
210269
+ void __fastpack17(const uint32_t *__restrict in, uint32_t *__restrict out) {
210270
+ Unroller<17>::Pack(in, out);
210271
+ }
210272
+
210273
+ void __fastpack18(const uint32_t *__restrict in, uint32_t *__restrict out) {
210274
+ Unroller<18>::Pack(in, out);
210275
+ }
210276
+
210277
+ void __fastpack19(const uint32_t *__restrict in, uint32_t *__restrict out) {
210278
+ Unroller<19>::Pack(in, out);
210279
+ }
210280
+
210281
+ void __fastpack20(const uint32_t *__restrict in, uint32_t *__restrict out) {
210282
+ Unroller<20>::Pack(in, out);
210283
+ }
210284
+
210285
+ void __fastpack21(const uint32_t *__restrict in, uint32_t *__restrict out) {
210286
+ Unroller<21>::Pack(in, out);
210287
+ }
210288
+
210289
+ void __fastpack22(const uint32_t *__restrict in, uint32_t *__restrict out) {
210290
+ Unroller<22>::Pack(in, out);
210291
+ }
210292
+
210293
+ void __fastpack23(const uint32_t *__restrict in, uint32_t *__restrict out) {
210294
+ Unroller<23>::Pack(in, out);
210295
+ }
210296
+
210297
+ void __fastpack24(const uint32_t *__restrict in, uint32_t *__restrict out) {
210298
+ Unroller<24>::Pack(in, out);
210299
+ }
210300
+
210301
+ void __fastpack25(const uint32_t *__restrict in, uint32_t *__restrict out) {
210302
+ Unroller<25>::Pack(in, out);
210303
+ }
210304
+
210305
+ void __fastpack26(const uint32_t *__restrict in, uint32_t *__restrict out) {
210306
+ Unroller<26>::Pack(in, out);
210307
+ }
210308
+
210309
+ void __fastpack27(const uint32_t *__restrict in, uint32_t *__restrict out) {
210310
+ Unroller<27>::Pack(in, out);
210311
+ }
210312
+
210313
+ void __fastpack28(const uint32_t *__restrict in, uint32_t *__restrict out) {
210314
+ Unroller<28>::Pack(in, out);
210315
+ }
210316
+
210317
+ void __fastpack29(const uint32_t *__restrict in, uint32_t *__restrict out) {
210318
+ Unroller<29>::Pack(in, out);
210319
+ }
210320
+
210321
+ void __fastpack30(const uint32_t *__restrict in, uint32_t *__restrict out) {
210322
+ Unroller<30>::Pack(in, out);
210323
+ }
210324
+
210325
+ void __fastpack31(const uint32_t *__restrict in, uint32_t *__restrict out) {
210326
+ Unroller<31>::Pack(in, out);
210327
+ }
210328
+
210329
+ void __fastpack32(const uint32_t *__restrict in, uint32_t *__restrict out) {
210330
+ for (int k = 0; k < 32; ++k)
210331
+ out[k] = in[k];
210332
+ }
210333
+
210334
+ // fastpack for 64 bits
210335
+
210336
+ void __fastpack1(const uint64_t *__restrict in, uint32_t *__restrict out) {
210337
+ Unroller<1>::Pack(in, out);
210338
+ }
210339
+
210340
+ void __fastpack2(const uint64_t *__restrict in, uint32_t *__restrict out) {
210341
+ Unroller<2>::Pack(in, out);
210342
+ }
210343
+
210344
+ void __fastpack3(const uint64_t *__restrict in, uint32_t *__restrict out) {
210345
+ Unroller<3>::Pack(in, out);
210346
+ }
210347
+
210348
+ void __fastpack4(const uint64_t *__restrict in, uint32_t *__restrict out) {
210349
+ Unroller<4>::Pack(in, out);
210350
+ }
210351
+
210352
+ void __fastpack5(const uint64_t *__restrict in, uint32_t *__restrict out) {
210353
+ Unroller<5>::Pack(in, out);
210354
+ }
210355
+
210356
+ void __fastpack6(const uint64_t *__restrict in, uint32_t *__restrict out) {
210357
+ Unroller<6>::Pack(in, out);
210358
+ }
210359
+
210360
+ void __fastpack7(const uint64_t *__restrict in, uint32_t *__restrict out) {
210361
+ Unroller<7>::Pack(in, out);
210362
+ }
210363
+
210364
+ void __fastpack8(const uint64_t *__restrict in, uint32_t *__restrict out) {
210365
+ Unroller<8>::Pack(in, out);
210366
+ }
210367
+
210368
+ void __fastpack9(const uint64_t *__restrict in, uint32_t *__restrict out) {
210369
+ Unroller<9>::Pack(in, out);
210370
+ }
210371
+
210372
+ void __fastpack10(const uint64_t *__restrict in, uint32_t *__restrict out) {
210373
+ Unroller<10>::Pack(in, out);
210374
+ }
210375
+
210376
+ void __fastpack11(const uint64_t *__restrict in, uint32_t *__restrict out) {
210377
+ Unroller<11>::Pack(in, out);
210378
+ }
210379
+
210380
+ void __fastpack12(const uint64_t *__restrict in, uint32_t *__restrict out) {
210381
+ Unroller<12>::Pack(in, out);
210382
+ }
210383
+
210384
+ void __fastpack13(const uint64_t *__restrict in, uint32_t *__restrict out) {
210385
+ Unroller<13>::Pack(in, out);
210386
+ }
210387
+
210388
+ void __fastpack14(const uint64_t *__restrict in, uint32_t *__restrict out) {
210389
+ Unroller<14>::Pack(in, out);
210390
+ }
210391
+
210392
+ void __fastpack15(const uint64_t *__restrict in, uint32_t *__restrict out) {
210393
+ Unroller<15>::Pack(in, out);
210394
+ }
210395
+
210396
+ void __fastpack16(const uint64_t *__restrict in, uint32_t *__restrict out) {
210397
+ Unroller<16>::Pack(in, out);
210398
+ }
210399
+
210400
+ void __fastpack17(const uint64_t *__restrict in, uint32_t *__restrict out) {
210401
+ Unroller<17>::Pack(in, out);
210402
+ }
210403
+
210404
+ void __fastpack18(const uint64_t *__restrict in, uint32_t *__restrict out) {
210405
+ Unroller<18>::Pack(in, out);
210406
+ }
210407
+
210408
+ void __fastpack19(const uint64_t *__restrict in, uint32_t *__restrict out) {
210409
+ Unroller<19>::Pack(in, out);
210410
+ }
210411
+
210412
+ void __fastpack20(const uint64_t *__restrict in, uint32_t *__restrict out) {
210413
+ Unroller<20>::Pack(in, out);
210414
+ }
210415
+
210416
+ void __fastpack21(const uint64_t *__restrict in, uint32_t *__restrict out) {
210417
+ Unroller<21>::Pack(in, out);
210418
+ }
210419
+
210420
+ void __fastpack22(const uint64_t *__restrict in, uint32_t *__restrict out) {
210421
+ Unroller<22>::Pack(in, out);
210422
+ }
210423
+
210424
+ void __fastpack23(const uint64_t *__restrict in, uint32_t *__restrict out) {
210425
+ Unroller<23>::Pack(in, out);
210426
+ }
210427
+
210428
+ void __fastpack24(const uint64_t *__restrict in, uint32_t *__restrict out) {
210429
+ Unroller<24>::Pack(in, out);
210430
+ }
210431
+
210432
+ void __fastpack25(const uint64_t *__restrict in, uint32_t *__restrict out) {
210433
+ Unroller<25>::Pack(in, out);
210434
+ }
210435
+
210436
+ void __fastpack26(const uint64_t *__restrict in, uint32_t *__restrict out) {
210437
+ Unroller<26>::Pack(in, out);
210438
+ }
210439
+
210440
+ void __fastpack27(const uint64_t *__restrict in, uint32_t *__restrict out) {
210441
+ Unroller<27>::Pack(in, out);
210442
+ }
210443
+
210444
+ void __fastpack28(const uint64_t *__restrict in, uint32_t *__restrict out) {
210445
+ Unroller<28>::Pack(in, out);
210446
+ }
210447
+
210448
+ void __fastpack29(const uint64_t *__restrict in, uint32_t *__restrict out) {
210449
+ Unroller<29>::Pack(in, out);
210450
+ }
210451
+
210452
+ void __fastpack30(const uint64_t *__restrict in, uint32_t *__restrict out) {
210453
+ Unroller<30>::Pack(in, out);
210454
+ }
210455
+
210456
+ void __fastpack31(const uint64_t *__restrict in, uint32_t *__restrict out) {
210457
+ Unroller<31>::Pack(in, out);
210458
+ }
210459
+
210460
+ void __fastpack32(const uint64_t *__restrict in, uint32_t *__restrict out) {
210461
+ for (int k = 0; k < 32; ++k) {
210462
+ out[k] = static_cast<uint32_t>(in[k]);
210463
+ }
210464
+ }
210465
+
210466
+ void __fastpack33(const uint64_t *__restrict in, uint32_t *__restrict out) {
210467
+ Unroller<33>::Pack(in, out);
210468
+ }
210469
+
210470
+ void __fastpack34(const uint64_t *__restrict in, uint32_t *__restrict out) {
210471
+ Unroller<34>::Pack(in, out);
210472
+ }
210473
+
210474
+ void __fastpack35(const uint64_t *__restrict in, uint32_t *__restrict out) {
210475
+ Unroller<35>::Pack(in, out);
210476
+ }
210477
+
210478
+ void __fastpack36(const uint64_t *__restrict in, uint32_t *__restrict out) {
210479
+ Unroller<36>::Pack(in, out);
210480
+ }
210481
+
210482
+ void __fastpack37(const uint64_t *__restrict in, uint32_t *__restrict out) {
210483
+ Unroller<37>::Pack(in, out);
210484
+ }
210485
+
210486
+ void __fastpack38(const uint64_t *__restrict in, uint32_t *__restrict out) {
210487
+ Unroller<38>::Pack(in, out);
210488
+ }
210489
+
210490
+ void __fastpack39(const uint64_t *__restrict in, uint32_t *__restrict out) {
210491
+ Unroller<39>::Pack(in, out);
210492
+ }
210493
+
210494
+ void __fastpack40(const uint64_t *__restrict in, uint32_t *__restrict out) {
210495
+ Unroller<40>::Pack(in, out);
210496
+ }
210497
+
210498
+ void __fastpack41(const uint64_t *__restrict in, uint32_t *__restrict out) {
210499
+ Unroller<41>::Pack(in, out);
210500
+ }
210501
+
210502
+ void __fastpack42(const uint64_t *__restrict in, uint32_t *__restrict out) {
210503
+ Unroller<42>::Pack(in, out);
210504
+ }
210505
+
210506
+ void __fastpack43(const uint64_t *__restrict in, uint32_t *__restrict out) {
210507
+ Unroller<43>::Pack(in, out);
210508
+ }
210509
+
210510
+ void __fastpack44(const uint64_t *__restrict in, uint32_t *__restrict out) {
210511
+ Unroller<44>::Pack(in, out);
210512
+ }
210513
+
210514
+ void __fastpack45(const uint64_t *__restrict in, uint32_t *__restrict out) {
210515
+ Unroller<45>::Pack(in, out);
210516
+ }
210517
+
210518
+ void __fastpack46(const uint64_t *__restrict in, uint32_t *__restrict out) {
210519
+ Unroller<46>::Pack(in, out);
210520
+ }
210521
+
210522
+ void __fastpack47(const uint64_t *__restrict in, uint32_t *__restrict out) {
210523
+ Unroller<47>::Pack(in, out);
210524
+ }
210525
+
210526
+ void __fastpack48(const uint64_t *__restrict in, uint32_t *__restrict out) {
210527
+ Unroller<48>::Pack(in, out);
210528
+ }
210529
+
210530
+ void __fastpack49(const uint64_t *__restrict in, uint32_t *__restrict out) {
210531
+ Unroller<49>::Pack(in, out);
210532
+ }
210533
+
210534
+ void __fastpack50(const uint64_t *__restrict in, uint32_t *__restrict out) {
210535
+ Unroller<50>::Pack(in, out);
210536
+ }
210537
+
210538
+ void __fastpack51(const uint64_t *__restrict in, uint32_t *__restrict out) {
210539
+ Unroller<51>::Pack(in, out);
210540
+ }
210541
+
210542
+ void __fastpack52(const uint64_t *__restrict in, uint32_t *__restrict out) {
210543
+ Unroller<52>::Pack(in, out);
210544
+ }
210545
+
210546
+ void __fastpack53(const uint64_t *__restrict in, uint32_t *__restrict out) {
210547
+ Unroller<53>::Pack(in, out);
210548
+ }
210549
+
210550
+ void __fastpack54(const uint64_t *__restrict in, uint32_t *__restrict out) {
210551
+ Unroller<54>::Pack(in, out);
210552
+ }
210553
+
210554
+ void __fastpack55(const uint64_t *__restrict in, uint32_t *__restrict out) {
210555
+ Unroller<55>::Pack(in, out);
210556
+ }
210557
+
210558
+ void __fastpack56(const uint64_t *__restrict in, uint32_t *__restrict out) {
210559
+ Unroller<56>::Pack(in, out);
210560
+ }
210561
+
210562
+ void __fastpack57(const uint64_t *__restrict in, uint32_t *__restrict out) {
210563
+ Unroller<57>::Pack(in, out);
210564
+ }
210565
+
210566
+ void __fastpack58(const uint64_t *__restrict in, uint32_t *__restrict out) {
210567
+ Unroller<58>::Pack(in, out);
210568
+ }
210569
+
210570
+ void __fastpack59(const uint64_t *__restrict in, uint32_t *__restrict out) {
210571
+ Unroller<59>::Pack(in, out);
210572
+ }
210573
+
210574
+ void __fastpack60(const uint64_t *__restrict in, uint32_t *__restrict out) {
210575
+ Unroller<60>::Pack(in, out);
210576
+ }
210577
+
210578
+ void __fastpack61(const uint64_t *__restrict in, uint32_t *__restrict out) {
210579
+ Unroller<61>::Pack(in, out);
210580
+ }
210581
+
210582
+ void __fastpack62(const uint64_t *__restrict in, uint32_t *__restrict out) {
210583
+ Unroller<62>::Pack(in, out);
210584
+ }
210585
+
210586
+ void __fastpack63(const uint64_t *__restrict in, uint32_t *__restrict out) {
210587
+ Unroller<63>::Pack(in, out);
210588
+ }
210589
+
210590
+ void __fastpack64(const uint64_t *__restrict in, uint32_t *__restrict out) {
210591
+ for (int i = 0; i < 32; ++i) {
210592
+ out[2 * i] = static_cast<uint32_t>(in[i]);
210593
+ out[2 * i + 1] = in[i] >> 32;
210594
+ }
210595
+ }
210596
+ } // namespace internal
210597
+ } // namespace duckdb_fastpforlib
210598
+
210599
+
210600
+ // LICENSE_CHANGE_END
210601
+
210602
+
207504
210603
  // LICENSE_CHANGE_BEGIN
207505
210604
  // The following code up to LICENSE_CHANGE_END is subject to THIRD PARTY LICENSE #2
207506
210605
  // See the end of this file for a list
@@ -257634,4 +260733,199 @@ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
257634
260733
  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
257635
260734
  POSSIBILITY OF SUCH DAMAGE.
257636
260735
 
260736
+
260737
+ ### THIRD PARTY LICENSE #10 ###
260738
+
260739
+ Apache License
260740
+ Version 2.0, January 2004
260741
+ http://www.apache.org/licenses/
260742
+
260743
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
260744
+
260745
+ 1. Definitions.
260746
+
260747
+ "License" shall mean the terms and conditions for use, reproduction, and
260748
+ distribution as defined by Sections 1 through 9 of this document.
260749
+
260750
+ "Licensor" shall mean the copyright owner or entity authorized by the copyright
260751
+ owner that is granting the License.
260752
+
260753
+ "Legal Entity" shall mean the union of the acting entity and all other entities
260754
+ that control, are controlled by, or are under common control with that entity.
260755
+ For the purposes of this definition, "control" means (i) the power, direct or
260756
+ indirect, to cause the direction or management of such entity, whether by
260757
+ contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the
260758
+ outstanding shares, or (iii) beneficial ownership of such entity.
260759
+
260760
+ "You" (or "Your") shall mean an individual or Legal Entity exercising
260761
+ permissions granted by this License.
260762
+
260763
+ "Source" form shall mean the preferred form for making modifications, including
260764
+ but not limited to software source code, documentation source, and configuration
260765
+ files.
260766
+
260767
+ "Object" form shall mean any form resulting from mechanical transformation or
260768
+ translation of a Source form, including but not limited to compiled object code,
260769
+ generated documentation, and conversions to other media types.
260770
+
260771
+ "Work" shall mean the work of authorship, whether in Source or Object form, made
260772
+ available under the License, as indicated by a copyright notice that is included
260773
+ in or attached to the work (an example is provided in the Appendix below).
260774
+
260775
+ "Derivative Works" shall mean any work, whether in Source or Object form, that
260776
+ is based on (or derived from) the Work and for which the editorial revisions,
260777
+ annotations, elaborations, or other modifications represent, as a whole, an
260778
+ original work of authorship. For the purposes of this License, Derivative Works
260779
+ shall not include works that remain separable from, or merely link (or bind by
260780
+ name) to the interfaces of, the Work and Derivative Works thereof.
260781
+
260782
+ "Contribution" shall mean any work of authorship, including the original version
260783
+ of the Work and any modifications or additions to that Work or Derivative Works
260784
+ thereof, that is intentionally submitted to Licensor for inclusion in the Work
260785
+ by the copyright owner or by an individual or Legal Entity authorized to submit
260786
+ on behalf of the copyright owner. For the purposes of this definition,
260787
+ "submitted" means any form of electronic, verbal, or written communication sent
260788
+ to the Licensor or its representatives, including but not limited to
260789
+ communication on electronic mailing lists, source code control systems, and
260790
+ issue tracking systems that are managed by, or on behalf of, the Licensor for
260791
+ the purpose of discussing and improving the Work, but excluding communication
260792
+ that is conspicuously marked or otherwise designated in writing by the copyright
260793
+ owner as "Not a Contribution."
260794
+
260795
+ "Contributor" shall mean Licensor and any individual or Legal Entity on behalf
260796
+ of whom a Contribution has been received by Licensor and subsequently
260797
+ incorporated within the Work.
260798
+
260799
+ 2. Grant of Copyright License.
260800
+
260801
+ Subject to the terms and conditions of this License, each Contributor hereby
260802
+ grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
260803
+ irrevocable copyright license to reproduce, prepare Derivative Works of,
260804
+ publicly display, publicly perform, sublicense, and distribute the Work and such
260805
+ Derivative Works in Source or Object form.
260806
+
260807
+ 3. Grant of Patent License.
260808
+
260809
+ Subject to the terms and conditions of this License, each Contributor hereby
260810
+ grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
260811
+ irrevocable (except as stated in this section) patent license to make, have
260812
+ made, use, offer to sell, sell, import, and otherwise transfer the Work, where
260813
+ such license applies only to those patent claims licensable by such Contributor
260814
+ that are necessarily infringed by their Contribution(s) alone or by combination
260815
+ of their Contribution(s) with the Work to which such Contribution(s) was
260816
+ submitted. If You institute patent litigation against any entity (including a
260817
+ cross-claim or counterclaim in a lawsuit) alleging that the Work or a
260818
+ Contribution incorporated within the Work constitutes direct or contributory
260819
+ patent infringement, then any patent licenses granted to You under this License
260820
+ for that Work shall terminate as of the date such litigation is filed.
260821
+
260822
+ 4. Redistribution.
260823
+
260824
+ You may reproduce and distribute copies of the Work or Derivative Works thereof
260825
+ in any medium, with or without modifications, and in Source or Object form,
260826
+ provided that You meet the following conditions:
260827
+
260828
+ You must give any other recipients of the Work or Derivative Works a copy of
260829
+ this License; and
260830
+ You must cause any modified files to carry prominent notices stating that You
260831
+ changed the files; and
260832
+ You must retain, in the Source form of any Derivative Works that You distribute,
260833
+ all copyright, patent, trademark, and attribution notices from the Source form
260834
+ of the Work, excluding those notices that do not pertain to any part of the
260835
+ Derivative Works; and
260836
+ If the Work includes a "NOTICE" text file as part of its distribution, then any
260837
+ Derivative Works that You distribute must include a readable copy of the
260838
+ attribution notices contained within such NOTICE file, excluding those notices
260839
+ that do not pertain to any part of the Derivative Works, in at least one of the
260840
+ following places: within a NOTICE text file distributed as part of the
260841
+ Derivative Works; within the Source form or documentation, if provided along
260842
+ with the Derivative Works; or, within a display generated by the Derivative
260843
+ Works, if and wherever such third-party notices normally appear. The contents of
260844
+ the NOTICE file are for informational purposes only and do not modify the
260845
+ License. You may add Your own attribution notices within Derivative Works that
260846
+ You distribute, alongside or as an addendum to the NOTICE text from the Work,
260847
+ provided that such additional attribution notices cannot be construed as
260848
+ modifying the License.
260849
+ You may add Your own copyright statement to Your modifications and may provide
260850
+ additional or different license terms and conditions for use, reproduction, or
260851
+ distribution of Your modifications, or for any such Derivative Works as a whole,
260852
+ provided Your use, reproduction, and distribution of the Work otherwise complies
260853
+ with the conditions stated in this License.
260854
+
260855
+ 5. Submission of Contributions.
260856
+
260857
+ Unless You explicitly state otherwise, any Contribution intentionally submitted
260858
+ for inclusion in the Work by You to the Licensor shall be under the terms and
260859
+ conditions of this License, without any additional terms or conditions.
260860
+ Notwithstanding the above, nothing herein shall supersede or modify the terms of
260861
+ any separate license agreement you may have executed with Licensor regarding
260862
+ such Contributions.
260863
+
260864
+ 6. Trademarks.
260865
+
260866
+ This License does not grant permission to use the trade names, trademarks,
260867
+ service marks, or product names of the Licensor, except as required for
260868
+ reasonable and customary use in describing the origin of the Work and
260869
+ reproducing the content of the NOTICE file.
260870
+
260871
+ 7. Disclaimer of Warranty.
260872
+
260873
+ Unless required by applicable law or agreed to in writing, Licensor provides the
260874
+ Work (and each Contributor provides its Contributions) on an "AS IS" BASIS,
260875
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied,
260876
+ including, without limitation, any warranties or conditions of TITLE,
260877
+ NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are
260878
+ solely responsible for determining the appropriateness of using or
260879
+ redistributing the Work and assume any risks associated with Your exercise of
260880
+ permissions under this License.
260881
+
260882
+ 8. Limitation of Liability.
260883
+
260884
+ In no event and under no legal theory, whether in tort (including negligence),
260885
+ contract, or otherwise, unless required by applicable law (such as deliberate
260886
+ and grossly negligent acts) or agreed to in writing, shall any Contributor be
260887
+ liable to You for damages, including any direct, indirect, special, incidental,
260888
+ or consequential damages of any character arising as a result of this License or
260889
+ out of the use or inability to use the Work (including but not limited to
260890
+ damages for loss of goodwill, work stoppage, computer failure or malfunction, or
260891
+ any and all other commercial damages or losses), even if such Contributor has
260892
+ been advised of the possibility of such damages.
260893
+
260894
+ 9. Accepting Warranty or Additional Liability.
260895
+
260896
+ While redistributing the Work or Derivative Works thereof, You may choose to
260897
+ offer, and charge a fee for, acceptance of support, warranty, indemnity, or
260898
+ other liability obligations and/or rights consistent with this License. However,
260899
+ in accepting such obligations, You may act only on Your own behalf and on Your
260900
+ sole responsibility, not on behalf of any other Contributor, and only if You
260901
+ agree to indemnify, defend, and hold each Contributor harmless for any liability
260902
+ incurred by, or claims asserted against, such Contributor by reason of your
260903
+ accepting any such warranty or additional liability.
260904
+
260905
+ END OF TERMS AND CONDITIONS
260906
+
260907
+ APPENDIX: How to apply the Apache License to your work
260908
+
260909
+ To apply the Apache License to your work, attach the following boilerplate
260910
+ notice, with the fields enclosed by brackets "[]" replaced with your own
260911
+ identifying information. (Don't include the brackets!) The text should be
260912
+ enclosed in the appropriate comment syntax for the file format. We also
260913
+ recommend that a file or class name and description of purpose be included on
260914
+ the same "printed page" as the copyright notice for easier identification within
260915
+ third-party archives.
260916
+
260917
+ Copyright [yyyy] [name of copyright owner]
260918
+
260919
+ Licensed under the Apache License, Version 2.0 (the "License");
260920
+ you may not use this file except in compliance with the License.
260921
+ You may obtain a copy of the License at
260922
+
260923
+ http://www.apache.org/licenses/LICENSE-2.0
260924
+
260925
+ Unless required by applicable law or agreed to in writing, software
260926
+ distributed under the License is distributed on an "AS IS" BASIS,
260927
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
260928
+ See the License for the specific language governing permissions and
260929
+ limitations under the License.
260930
+
257637
260931
  */