duckdb 0.3.2-dev396.0 → 0.3.2-dev454.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/duckdb.cpp +3329 -35
- package/src/duckdb.hpp +2 -2
package/src/duckdb.cpp
CHANGED
|
@@ -80621,6 +80621,11 @@ struct RLEFun {
|
|
|
80621
80621
|
static bool TypeIsSupported(PhysicalType type);
|
|
80622
80622
|
};
|
|
80623
80623
|
|
|
80624
|
+
struct BitpackingFun {
|
|
80625
|
+
static CompressionFunction GetFunction(PhysicalType type);
|
|
80626
|
+
static bool TypeIsSupported(PhysicalType type);
|
|
80627
|
+
};
|
|
80628
|
+
|
|
80624
80629
|
} // namespace duckdb
|
|
80625
80630
|
|
|
80626
80631
|
|
|
@@ -80640,6 +80645,7 @@ static DefaultCompressionMethod internal_compression_methods[] = {
|
|
|
80640
80645
|
{CompressionType::COMPRESSION_CONSTANT, ConstantFun::GetFunction, ConstantFun::TypeIsSupported},
|
|
80641
80646
|
{CompressionType::COMPRESSION_UNCOMPRESSED, UncompressedFun::GetFunction, UncompressedFun::TypeIsSupported},
|
|
80642
80647
|
{CompressionType::COMPRESSION_RLE, RLEFun::GetFunction, RLEFun::TypeIsSupported},
|
|
80648
|
+
{CompressionType::COMPRESSION_BITPACKING, BitpackingFun::GetFunction, BitpackingFun::TypeIsSupported},
|
|
80643
80649
|
{CompressionType::COMPRESSION_AUTO, nullptr, nullptr}};
|
|
80644
80650
|
|
|
80645
80651
|
static CompressionFunction *FindCompressionFunction(CompressionFunctionSet &set, CompressionType type,
|
|
@@ -80688,6 +80694,7 @@ vector<CompressionFunction *> DBConfig::GetCompressionFunctions(PhysicalType dat
|
|
|
80688
80694
|
vector<CompressionFunction *> result;
|
|
80689
80695
|
TryLoadCompression(*this, result, CompressionType::COMPRESSION_UNCOMPRESSED, data_type);
|
|
80690
80696
|
TryLoadCompression(*this, result, CompressionType::COMPRESSION_RLE, data_type);
|
|
80697
|
+
TryLoadCompression(*this, result, CompressionType::COMPRESSION_BITPACKING, data_type);
|
|
80691
80698
|
return result;
|
|
80692
80699
|
}
|
|
80693
80700
|
|
|
@@ -159282,56 +159289,1170 @@ void PartialBlock::FlushToDisk(DatabaseInstance &db) {
|
|
|
159282
159289
|
}
|
|
159283
159290
|
|
|
159284
159291
|
} // namespace duckdb
|
|
159285
|
-
//===----------------------------------------------------------------------===//
|
|
159286
|
-
// DuckDB
|
|
159287
|
-
//
|
|
159288
|
-
// duckdb/storage/segment/uncompressed.hpp
|
|
159289
|
-
//
|
|
159290
|
-
//
|
|
159291
|
-
//===----------------------------------------------------------------------===//
|
|
159292
159292
|
|
|
159293
159293
|
|
|
159294
|
+
// LICENSE_CHANGE_BEGIN
|
|
159295
|
+
// The following code up to LICENSE_CHANGE_END is subject to THIRD PARTY LICENSE #10
|
|
159296
|
+
// See the end of this file for a list
|
|
159297
|
+
|
|
159298
|
+
/**
|
|
159299
|
+
* This code is released under the
|
|
159300
|
+
* Apache License Version 2.0 http://www.apache.org/licenses/.
|
|
159301
|
+
*
|
|
159302
|
+
* (c) Daniel Lemire, http://lemire.me/en/
|
|
159303
|
+
*/
|
|
159304
|
+
|
|
159294
159305
|
|
|
159295
159306
|
|
|
159307
|
+
// LICENSE_CHANGE_BEGIN
|
|
159308
|
+
// The following code up to LICENSE_CHANGE_END is subject to THIRD PARTY LICENSE #10
|
|
159309
|
+
// See the end of this file for a list
|
|
159296
159310
|
|
|
159297
|
-
|
|
159298
|
-
|
|
159311
|
+
/**
|
|
159312
|
+
* This code is released under the
|
|
159313
|
+
* Apache License Version 2.0 http://www.apache.org/licenses/.
|
|
159314
|
+
*
|
|
159315
|
+
* (c) Daniel Lemire, http://fastpforlib.me/en/
|
|
159316
|
+
*/
|
|
159299
159317
|
|
|
159300
|
-
|
|
159301
|
-
|
|
159302
|
-
|
|
159303
|
-
|
|
159304
|
-
|
|
159305
|
-
|
|
159318
|
+
#include <cinttypes>
|
|
159319
|
+
#include <string>
|
|
159320
|
+
|
|
159321
|
+
namespace duckdb_fastpforlib {
|
|
159322
|
+
namespace internal {
|
|
159323
|
+
|
|
159324
|
+
// Unpacks 8 uint8_t values
|
|
159325
|
+
void __fastunpack0(const uint8_t *__restrict in, uint8_t *__restrict out);
|
|
159326
|
+
void __fastunpack1(const uint8_t *__restrict in, uint8_t *__restrict out);
|
|
159327
|
+
void __fastunpack2(const uint8_t *__restrict in, uint8_t *__restrict out);
|
|
159328
|
+
void __fastunpack3(const uint8_t *__restrict in, uint8_t *__restrict out);
|
|
159329
|
+
void __fastunpack4(const uint8_t *__restrict in, uint8_t *__restrict out);
|
|
159330
|
+
void __fastunpack5(const uint8_t *__restrict in, uint8_t *__restrict out);
|
|
159331
|
+
void __fastunpack6(const uint8_t *__restrict in, uint8_t *__restrict out);
|
|
159332
|
+
void __fastunpack7(const uint8_t *__restrict in, uint8_t *__restrict out);
|
|
159333
|
+
void __fastunpack8(const uint8_t *__restrict in, uint8_t *__restrict out);
|
|
159334
|
+
|
|
159335
|
+
// Unpacks 16 uint16_t values
|
|
159336
|
+
void __fastunpack0(const uint16_t *__restrict in, uint16_t *__restrict out);
|
|
159337
|
+
void __fastunpack1(const uint16_t *__restrict in, uint16_t *__restrict out);
|
|
159338
|
+
void __fastunpack2(const uint16_t *__restrict in, uint16_t *__restrict out);
|
|
159339
|
+
void __fastunpack3(const uint16_t *__restrict in, uint16_t *__restrict out);
|
|
159340
|
+
void __fastunpack4(const uint16_t *__restrict in, uint16_t *__restrict out);
|
|
159341
|
+
void __fastunpack5(const uint16_t *__restrict in, uint16_t *__restrict out);
|
|
159342
|
+
void __fastunpack6(const uint16_t *__restrict in, uint16_t *__restrict out);
|
|
159343
|
+
void __fastunpack7(const uint16_t *__restrict in, uint16_t *__restrict out);
|
|
159344
|
+
void __fastunpack8(const uint16_t *__restrict in, uint16_t *__restrict out);
|
|
159345
|
+
void __fastunpack9(const uint16_t *__restrict in, uint16_t *__restrict out);
|
|
159346
|
+
void __fastunpack10(const uint16_t *__restrict in, uint16_t *__restrict out);
|
|
159347
|
+
void __fastunpack11(const uint16_t *__restrict in, uint16_t *__restrict out);
|
|
159348
|
+
void __fastunpack12(const uint16_t *__restrict in, uint16_t *__restrict out);
|
|
159349
|
+
void __fastunpack13(const uint16_t *__restrict in, uint16_t *__restrict out);
|
|
159350
|
+
void __fastunpack14(const uint16_t *__restrict in, uint16_t *__restrict out);
|
|
159351
|
+
void __fastunpack15(const uint16_t *__restrict in, uint16_t *__restrict out);
|
|
159352
|
+
void __fastunpack16(const uint16_t *__restrict in, uint16_t *__restrict out);
|
|
159353
|
+
|
|
159354
|
+
// Unpacks 32 uint32_t values
|
|
159355
|
+
void __fastunpack0(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159356
|
+
void __fastunpack1(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159357
|
+
void __fastunpack2(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159358
|
+
void __fastunpack3(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159359
|
+
void __fastunpack4(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159360
|
+
void __fastunpack5(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159361
|
+
void __fastunpack6(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159362
|
+
void __fastunpack7(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159363
|
+
void __fastunpack8(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159364
|
+
void __fastunpack9(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159365
|
+
void __fastunpack10(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159366
|
+
void __fastunpack11(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159367
|
+
void __fastunpack12(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159368
|
+
void __fastunpack13(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159369
|
+
void __fastunpack14(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159370
|
+
void __fastunpack15(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159371
|
+
void __fastunpack16(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159372
|
+
void __fastunpack17(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159373
|
+
void __fastunpack18(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159374
|
+
void __fastunpack19(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159375
|
+
void __fastunpack20(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159376
|
+
void __fastunpack21(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159377
|
+
void __fastunpack22(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159378
|
+
void __fastunpack23(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159379
|
+
void __fastunpack24(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159380
|
+
void __fastunpack25(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159381
|
+
void __fastunpack26(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159382
|
+
void __fastunpack27(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159383
|
+
void __fastunpack28(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159384
|
+
void __fastunpack29(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159385
|
+
void __fastunpack30(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159386
|
+
void __fastunpack31(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159387
|
+
void __fastunpack32(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159388
|
+
|
|
159389
|
+
// Unpacks 32 uint64_t values
|
|
159390
|
+
void __fastunpack0(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159391
|
+
void __fastunpack1(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159392
|
+
void __fastunpack2(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159393
|
+
void __fastunpack3(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159394
|
+
void __fastunpack4(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159395
|
+
void __fastunpack5(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159396
|
+
void __fastunpack6(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159397
|
+
void __fastunpack7(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159398
|
+
void __fastunpack8(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159399
|
+
void __fastunpack9(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159400
|
+
void __fastunpack10(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159401
|
+
void __fastunpack11(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159402
|
+
void __fastunpack12(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159403
|
+
void __fastunpack13(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159404
|
+
void __fastunpack14(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159405
|
+
void __fastunpack15(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159406
|
+
void __fastunpack16(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159407
|
+
void __fastunpack17(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159408
|
+
void __fastunpack18(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159409
|
+
void __fastunpack19(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159410
|
+
void __fastunpack20(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159411
|
+
void __fastunpack21(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159412
|
+
void __fastunpack22(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159413
|
+
void __fastunpack23(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159414
|
+
void __fastunpack24(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159415
|
+
void __fastunpack25(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159416
|
+
void __fastunpack26(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159417
|
+
void __fastunpack27(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159418
|
+
void __fastunpack28(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159419
|
+
void __fastunpack29(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159420
|
+
void __fastunpack30(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159421
|
+
void __fastunpack31(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159422
|
+
void __fastunpack32(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159423
|
+
void __fastunpack33(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159424
|
+
void __fastunpack34(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159425
|
+
void __fastunpack35(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159426
|
+
void __fastunpack36(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159427
|
+
void __fastunpack37(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159428
|
+
void __fastunpack38(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159429
|
+
void __fastunpack39(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159430
|
+
void __fastunpack40(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159431
|
+
void __fastunpack41(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159432
|
+
void __fastunpack42(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159433
|
+
void __fastunpack43(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159434
|
+
void __fastunpack44(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159435
|
+
void __fastunpack45(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159436
|
+
void __fastunpack46(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159437
|
+
void __fastunpack47(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159438
|
+
void __fastunpack48(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159439
|
+
void __fastunpack49(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159440
|
+
void __fastunpack50(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159441
|
+
void __fastunpack51(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159442
|
+
void __fastunpack52(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159443
|
+
void __fastunpack53(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159444
|
+
void __fastunpack54(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159445
|
+
void __fastunpack55(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159446
|
+
void __fastunpack56(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159447
|
+
void __fastunpack57(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159448
|
+
void __fastunpack58(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159449
|
+
void __fastunpack59(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159450
|
+
void __fastunpack60(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159451
|
+
void __fastunpack61(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159452
|
+
void __fastunpack62(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159453
|
+
void __fastunpack63(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159454
|
+
void __fastunpack64(const uint32_t *__restrict in, uint64_t *__restrict out);
|
|
159455
|
+
|
|
159456
|
+
// Packs 8 int8_t values
|
|
159457
|
+
void __fastpack0(const uint8_t *__restrict in, uint8_t *__restrict out);
|
|
159458
|
+
void __fastpack1(const uint8_t *__restrict in, uint8_t *__restrict out);
|
|
159459
|
+
void __fastpack2(const uint8_t *__restrict in, uint8_t *__restrict out);
|
|
159460
|
+
void __fastpack3(const uint8_t *__restrict in, uint8_t *__restrict out);
|
|
159461
|
+
void __fastpack4(const uint8_t *__restrict in, uint8_t *__restrict out);
|
|
159462
|
+
void __fastpack5(const uint8_t *__restrict in, uint8_t *__restrict out);
|
|
159463
|
+
void __fastpack6(const uint8_t *__restrict in, uint8_t *__restrict out);
|
|
159464
|
+
void __fastpack7(const uint8_t *__restrict in, uint8_t *__restrict out);
|
|
159465
|
+
void __fastpack8(const uint8_t *__restrict in, uint8_t *__restrict out);
|
|
159466
|
+
|
|
159467
|
+
// Packs 16 int16_t values
|
|
159468
|
+
void __fastpack0(const uint16_t *__restrict in, uint16_t *__restrict out);
|
|
159469
|
+
void __fastpack1(const uint16_t *__restrict in, uint16_t *__restrict out);
|
|
159470
|
+
void __fastpack2(const uint16_t *__restrict in, uint16_t *__restrict out);
|
|
159471
|
+
void __fastpack3(const uint16_t *__restrict in, uint16_t *__restrict out);
|
|
159472
|
+
void __fastpack4(const uint16_t *__restrict in, uint16_t *__restrict out);
|
|
159473
|
+
void __fastpack5(const uint16_t *__restrict in, uint16_t *__restrict out);
|
|
159474
|
+
void __fastpack6(const uint16_t *__restrict in, uint16_t *__restrict out);
|
|
159475
|
+
void __fastpack7(const uint16_t *__restrict in, uint16_t *__restrict out);
|
|
159476
|
+
void __fastpack8(const uint16_t *__restrict in, uint16_t *__restrict out);
|
|
159477
|
+
void __fastpack9(const uint16_t *__restrict in, uint16_t *__restrict out);
|
|
159478
|
+
void __fastpack10(const uint16_t *__restrict in, uint16_t *__restrict out);
|
|
159479
|
+
void __fastpack11(const uint16_t *__restrict in, uint16_t *__restrict out);
|
|
159480
|
+
void __fastpack12(const uint16_t *__restrict in, uint16_t *__restrict out);
|
|
159481
|
+
void __fastpack13(const uint16_t *__restrict in, uint16_t *__restrict out);
|
|
159482
|
+
void __fastpack14(const uint16_t *__restrict in, uint16_t *__restrict out);
|
|
159483
|
+
void __fastpack15(const uint16_t *__restrict in, uint16_t *__restrict out);
|
|
159484
|
+
void __fastpack16(const uint16_t *__restrict in, uint16_t *__restrict out);
|
|
159485
|
+
|
|
159486
|
+
// Packs 32 int32_t values
|
|
159487
|
+
void __fastpack0(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159488
|
+
void __fastpack1(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159489
|
+
void __fastpack2(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159490
|
+
void __fastpack3(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159491
|
+
void __fastpack4(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159492
|
+
void __fastpack5(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159493
|
+
void __fastpack6(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159494
|
+
void __fastpack7(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159495
|
+
void __fastpack8(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159496
|
+
void __fastpack9(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159497
|
+
void __fastpack10(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159498
|
+
void __fastpack11(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159499
|
+
void __fastpack12(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159500
|
+
void __fastpack13(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159501
|
+
void __fastpack14(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159502
|
+
void __fastpack15(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159503
|
+
void __fastpack16(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159504
|
+
void __fastpack17(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159505
|
+
void __fastpack18(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159506
|
+
void __fastpack19(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159507
|
+
void __fastpack20(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159508
|
+
void __fastpack21(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159509
|
+
void __fastpack22(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159510
|
+
void __fastpack23(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159511
|
+
void __fastpack24(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159512
|
+
void __fastpack25(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159513
|
+
void __fastpack26(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159514
|
+
void __fastpack27(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159515
|
+
void __fastpack28(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159516
|
+
void __fastpack29(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159517
|
+
void __fastpack30(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159518
|
+
void __fastpack31(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159519
|
+
void __fastpack32(const uint32_t *__restrict in, uint32_t *__restrict out);
|
|
159520
|
+
|
|
159521
|
+
// Packs 32 int64_t values
|
|
159522
|
+
void __fastpack0(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159523
|
+
void __fastpack1(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159524
|
+
void __fastpack2(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159525
|
+
void __fastpack3(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159526
|
+
void __fastpack4(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159527
|
+
void __fastpack5(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159528
|
+
void __fastpack6(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159529
|
+
void __fastpack7(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159530
|
+
void __fastpack8(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159531
|
+
void __fastpack9(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159532
|
+
void __fastpack10(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159533
|
+
void __fastpack11(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159534
|
+
void __fastpack12(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159535
|
+
void __fastpack13(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159536
|
+
void __fastpack14(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159537
|
+
void __fastpack15(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159538
|
+
void __fastpack16(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159539
|
+
void __fastpack17(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159540
|
+
void __fastpack18(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159541
|
+
void __fastpack19(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159542
|
+
void __fastpack20(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159543
|
+
void __fastpack21(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159544
|
+
void __fastpack22(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159545
|
+
void __fastpack23(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159546
|
+
void __fastpack24(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159547
|
+
void __fastpack25(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159548
|
+
void __fastpack26(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159549
|
+
void __fastpack27(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159550
|
+
void __fastpack28(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159551
|
+
void __fastpack29(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159552
|
+
void __fastpack30(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159553
|
+
void __fastpack31(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159554
|
+
void __fastpack32(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159555
|
+
void __fastpack33(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159556
|
+
void __fastpack34(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159557
|
+
void __fastpack35(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159558
|
+
void __fastpack36(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159559
|
+
void __fastpack37(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159560
|
+
void __fastpack38(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159561
|
+
void __fastpack39(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159562
|
+
void __fastpack40(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159563
|
+
void __fastpack41(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159564
|
+
void __fastpack42(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159565
|
+
void __fastpack43(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159566
|
+
void __fastpack44(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159567
|
+
void __fastpack45(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159568
|
+
void __fastpack46(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159569
|
+
void __fastpack47(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159570
|
+
void __fastpack48(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159571
|
+
void __fastpack49(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159572
|
+
void __fastpack50(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159573
|
+
void __fastpack51(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159574
|
+
void __fastpack52(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159575
|
+
void __fastpack53(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159576
|
+
void __fastpack54(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159577
|
+
void __fastpack55(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159578
|
+
void __fastpack56(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159579
|
+
void __fastpack57(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159580
|
+
void __fastpack58(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159581
|
+
void __fastpack59(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159582
|
+
void __fastpack60(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159583
|
+
void __fastpack61(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159584
|
+
void __fastpack62(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159585
|
+
void __fastpack63(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159586
|
+
void __fastpack64(const uint64_t *__restrict in, uint32_t *__restrict out);
|
|
159587
|
+
} // namespace internal
|
|
159588
|
+
} // namespace duckdb_fastpforlib
|
|
159589
|
+
|
|
159590
|
+
|
|
159591
|
+
// LICENSE_CHANGE_END
|
|
159592
|
+
|
|
159593
|
+
|
|
159594
|
+
#include <stdexcept>
|
|
159595
|
+
|
|
159596
|
+
namespace duckdb_fastpforlib {
|
|
159597
|
+
|
|
159598
|
+
namespace internal {
|
|
159599
|
+
|
|
159600
|
+
// Note that this only packs 8 values
|
|
159601
|
+
inline void fastunpack_quarter(const uint8_t *__restrict in, uint8_t *__restrict out, const uint32_t bit) {
|
|
159602
|
+
// Could have used function pointers instead of switch.
|
|
159603
|
+
// Switch calls do offer the compiler more opportunities for optimization in
|
|
159604
|
+
// theory. In this case, it makes no difference with a good compiler.
|
|
159605
|
+
switch (bit) {
|
|
159606
|
+
case 0:
|
|
159607
|
+
internal::__fastunpack0(in, out);
|
|
159608
|
+
break;
|
|
159609
|
+
case 1:
|
|
159610
|
+
internal::__fastunpack1(in, out);
|
|
159611
|
+
break;
|
|
159612
|
+
case 2:
|
|
159613
|
+
internal::__fastunpack2(in, out);
|
|
159614
|
+
break;
|
|
159615
|
+
case 3:
|
|
159616
|
+
internal::__fastunpack3(in, out);
|
|
159617
|
+
break;
|
|
159618
|
+
case 4:
|
|
159619
|
+
internal::__fastunpack4(in, out);
|
|
159620
|
+
break;
|
|
159621
|
+
case 5:
|
|
159622
|
+
internal::__fastunpack5(in, out);
|
|
159623
|
+
break;
|
|
159624
|
+
case 6:
|
|
159625
|
+
internal::__fastunpack6(in, out);
|
|
159626
|
+
break;
|
|
159627
|
+
case 7:
|
|
159628
|
+
internal::__fastunpack7(in, out);
|
|
159629
|
+
break;
|
|
159630
|
+
case 8:
|
|
159631
|
+
internal::__fastunpack8(in, out);
|
|
159632
|
+
break;
|
|
159633
|
+
default:
|
|
159634
|
+
throw std::logic_error("Invalid bit width for bitpacking");
|
|
159306
159635
|
}
|
|
159307
|
-
}
|
|
159636
|
+
}
|
|
159308
159637
|
|
|
159309
|
-
|
|
159310
|
-
|
|
159311
|
-
|
|
159638
|
+
// Note that this only packs 8 values
|
|
159639
|
+
inline void fastpack_quarter(const uint8_t *__restrict in, uint8_t *__restrict out, const uint32_t bit) {
|
|
159640
|
+
// Could have used function pointers instead of switch.
|
|
159641
|
+
// Switch calls do offer the compiler more opportunities for optimization in
|
|
159642
|
+
// theory. In this case, it makes no difference with a good compiler.
|
|
159643
|
+
switch (bit) {
|
|
159644
|
+
case 0:
|
|
159645
|
+
internal::__fastpack0(in, out);
|
|
159646
|
+
break;
|
|
159647
|
+
case 1:
|
|
159648
|
+
internal::__fastpack1(in, out);
|
|
159649
|
+
break;
|
|
159650
|
+
case 2:
|
|
159651
|
+
internal::__fastpack2(in, out);
|
|
159652
|
+
break;
|
|
159653
|
+
case 3:
|
|
159654
|
+
internal::__fastpack3(in, out);
|
|
159655
|
+
break;
|
|
159656
|
+
case 4:
|
|
159657
|
+
internal::__fastpack4(in, out);
|
|
159658
|
+
break;
|
|
159659
|
+
case 5:
|
|
159660
|
+
internal::__fastpack5(in, out);
|
|
159661
|
+
break;
|
|
159662
|
+
case 6:
|
|
159663
|
+
internal::__fastpack6(in, out);
|
|
159664
|
+
break;
|
|
159665
|
+
case 7:
|
|
159666
|
+
internal::__fastpack7(in, out);
|
|
159667
|
+
break;
|
|
159668
|
+
case 8:
|
|
159669
|
+
internal::__fastpack8(in, out);
|
|
159670
|
+
break;
|
|
159671
|
+
default:
|
|
159672
|
+
throw std::logic_error("Invalid bit width for bitpacking");
|
|
159673
|
+
}
|
|
159674
|
+
}
|
|
159312
159675
|
|
|
159313
|
-
|
|
159314
|
-
|
|
159315
|
-
|
|
159676
|
+
// Note that this only packs 16 values
|
|
159677
|
+
inline void fastunpack_half(const uint16_t *__restrict in, uint16_t *__restrict out, const uint32_t bit) {
|
|
159678
|
+
// Could have used function pointers instead of switch.
|
|
159679
|
+
// Switch calls do offer the compiler more opportunities for optimization in
|
|
159680
|
+
// theory. In this case, it makes no difference with a good compiler.
|
|
159681
|
+
switch (bit) {
|
|
159682
|
+
case 0:
|
|
159683
|
+
internal::__fastunpack0(in, out);
|
|
159684
|
+
break;
|
|
159685
|
+
case 1:
|
|
159686
|
+
internal::__fastunpack1(in, out);
|
|
159687
|
+
break;
|
|
159688
|
+
case 2:
|
|
159689
|
+
internal::__fastunpack2(in, out);
|
|
159690
|
+
break;
|
|
159691
|
+
case 3:
|
|
159692
|
+
internal::__fastunpack3(in, out);
|
|
159693
|
+
break;
|
|
159694
|
+
case 4:
|
|
159695
|
+
internal::__fastunpack4(in, out);
|
|
159696
|
+
break;
|
|
159697
|
+
case 5:
|
|
159698
|
+
internal::__fastunpack5(in, out);
|
|
159699
|
+
break;
|
|
159700
|
+
case 6:
|
|
159701
|
+
internal::__fastunpack6(in, out);
|
|
159702
|
+
break;
|
|
159703
|
+
case 7:
|
|
159704
|
+
internal::__fastunpack7(in, out);
|
|
159705
|
+
break;
|
|
159706
|
+
case 8:
|
|
159707
|
+
internal::__fastunpack8(in, out);
|
|
159708
|
+
break;
|
|
159709
|
+
case 9:
|
|
159710
|
+
internal::__fastunpack9(in, out);
|
|
159711
|
+
break;
|
|
159712
|
+
case 10:
|
|
159713
|
+
internal::__fastunpack10(in, out);
|
|
159714
|
+
break;
|
|
159715
|
+
case 11:
|
|
159716
|
+
internal::__fastunpack11(in, out);
|
|
159717
|
+
break;
|
|
159718
|
+
case 12:
|
|
159719
|
+
internal::__fastunpack12(in, out);
|
|
159720
|
+
break;
|
|
159721
|
+
case 13:
|
|
159722
|
+
internal::__fastunpack13(in, out);
|
|
159723
|
+
break;
|
|
159724
|
+
case 14:
|
|
159725
|
+
internal::__fastunpack14(in, out);
|
|
159726
|
+
break;
|
|
159727
|
+
case 15:
|
|
159728
|
+
internal::__fastunpack15(in, out);
|
|
159729
|
+
break;
|
|
159730
|
+
case 16:
|
|
159731
|
+
internal::__fastunpack16(in, out);
|
|
159732
|
+
break;
|
|
159733
|
+
default:
|
|
159734
|
+
throw std::logic_error("Invalid bit width for bitpacking");
|
|
159735
|
+
}
|
|
159736
|
+
}
|
|
159316
159737
|
|
|
159317
|
-
|
|
159318
|
-
|
|
159319
|
-
|
|
159320
|
-
|
|
159738
|
+
// Note that this only packs 16 values
|
|
159739
|
+
inline void fastpack_half(const uint16_t *__restrict in, uint16_t *__restrict out, const uint32_t bit) {
|
|
159740
|
+
// Could have used function pointers instead of switch.
|
|
159741
|
+
// Switch calls do offer the compiler more opportunities for optimization in
|
|
159742
|
+
// theory. In this case, it makes no difference with a good compiler.
|
|
159743
|
+
switch (bit) {
|
|
159744
|
+
case 0:
|
|
159745
|
+
internal::__fastpack0(in, out);
|
|
159746
|
+
break;
|
|
159747
|
+
case 1:
|
|
159748
|
+
internal::__fastpack1(in, out);
|
|
159749
|
+
break;
|
|
159750
|
+
case 2:
|
|
159751
|
+
internal::__fastpack2(in, out);
|
|
159752
|
+
break;
|
|
159753
|
+
case 3:
|
|
159754
|
+
internal::__fastpack3(in, out);
|
|
159755
|
+
break;
|
|
159756
|
+
case 4:
|
|
159757
|
+
internal::__fastpack4(in, out);
|
|
159758
|
+
break;
|
|
159759
|
+
case 5:
|
|
159760
|
+
internal::__fastpack5(in, out);
|
|
159761
|
+
break;
|
|
159762
|
+
case 6:
|
|
159763
|
+
internal::__fastpack6(in, out);
|
|
159764
|
+
break;
|
|
159765
|
+
case 7:
|
|
159766
|
+
internal::__fastpack7(in, out);
|
|
159767
|
+
break;
|
|
159768
|
+
case 8:
|
|
159769
|
+
internal::__fastpack8(in, out);
|
|
159770
|
+
break;
|
|
159771
|
+
case 9:
|
|
159772
|
+
internal::__fastpack9(in, out);
|
|
159773
|
+
break;
|
|
159774
|
+
case 10:
|
|
159775
|
+
internal::__fastpack10(in, out);
|
|
159776
|
+
break;
|
|
159777
|
+
case 11:
|
|
159778
|
+
internal::__fastpack11(in, out);
|
|
159779
|
+
break;
|
|
159780
|
+
case 12:
|
|
159781
|
+
internal::__fastpack12(in, out);
|
|
159782
|
+
break;
|
|
159783
|
+
case 13:
|
|
159784
|
+
internal::__fastpack13(in, out);
|
|
159785
|
+
break;
|
|
159786
|
+
case 14:
|
|
159787
|
+
internal::__fastpack14(in, out);
|
|
159788
|
+
break;
|
|
159789
|
+
case 15:
|
|
159790
|
+
internal::__fastpack15(in, out);
|
|
159791
|
+
break;
|
|
159792
|
+
case 16:
|
|
159793
|
+
internal::__fastpack16(in, out);
|
|
159794
|
+
break;
|
|
159795
|
+
default:
|
|
159796
|
+
throw std::logic_error("Invalid bit width for bitpacking");
|
|
159797
|
+
}
|
|
159798
|
+
}
|
|
159799
|
+
}
|
|
159321
159800
|
|
|
159322
|
-
|
|
159323
|
-
|
|
159324
|
-
|
|
159801
|
+
inline void fastunpack(const uint8_t *__restrict in, uint8_t *__restrict out, const uint32_t bit) {
|
|
159802
|
+
for (uint8_t i = 0; i < 4; i++) {
|
|
159803
|
+
internal::fastunpack_quarter(in + (i*bit), out+(i*8), bit);
|
|
159804
|
+
}
|
|
159805
|
+
}
|
|
159325
159806
|
|
|
159326
|
-
|
|
159327
|
-
|
|
159328
|
-
|
|
159329
|
-
|
|
159330
|
-
};
|
|
159807
|
+
inline void fastunpack(const uint16_t *__restrict in, uint16_t *__restrict out, const uint32_t bit) {
|
|
159808
|
+
internal::fastunpack_half(in, out, bit);
|
|
159809
|
+
internal::fastunpack_half(in + bit, out+16, bit);
|
|
159810
|
+
}
|
|
159331
159811
|
|
|
159332
|
-
|
|
159812
|
+
inline void fastunpack(const uint32_t *__restrict in,
|
|
159813
|
+
uint32_t *__restrict out, const uint32_t bit) {
|
|
159814
|
+
// Could have used function pointers instead of switch.
|
|
159815
|
+
// Switch calls do offer the compiler more opportunities for optimization in
|
|
159816
|
+
// theory. In this case, it makes no difference with a good compiler.
|
|
159817
|
+
switch (bit) {
|
|
159818
|
+
case 0:
|
|
159819
|
+
internal::__fastunpack0(in, out);
|
|
159820
|
+
break;
|
|
159821
|
+
case 1:
|
|
159822
|
+
internal::__fastunpack1(in, out);
|
|
159823
|
+
break;
|
|
159824
|
+
case 2:
|
|
159825
|
+
internal::__fastunpack2(in, out);
|
|
159826
|
+
break;
|
|
159827
|
+
case 3:
|
|
159828
|
+
internal::__fastunpack3(in, out);
|
|
159829
|
+
break;
|
|
159830
|
+
case 4:
|
|
159831
|
+
internal::__fastunpack4(in, out);
|
|
159832
|
+
break;
|
|
159833
|
+
case 5:
|
|
159834
|
+
internal::__fastunpack5(in, out);
|
|
159835
|
+
break;
|
|
159836
|
+
case 6:
|
|
159837
|
+
internal::__fastunpack6(in, out);
|
|
159838
|
+
break;
|
|
159839
|
+
case 7:
|
|
159840
|
+
internal::__fastunpack7(in, out);
|
|
159841
|
+
break;
|
|
159842
|
+
case 8:
|
|
159843
|
+
internal::__fastunpack8(in, out);
|
|
159844
|
+
break;
|
|
159845
|
+
case 9:
|
|
159846
|
+
internal::__fastunpack9(in, out);
|
|
159847
|
+
break;
|
|
159848
|
+
case 10:
|
|
159849
|
+
internal::__fastunpack10(in, out);
|
|
159850
|
+
break;
|
|
159851
|
+
case 11:
|
|
159852
|
+
internal::__fastunpack11(in, out);
|
|
159853
|
+
break;
|
|
159854
|
+
case 12:
|
|
159855
|
+
internal::__fastunpack12(in, out);
|
|
159856
|
+
break;
|
|
159857
|
+
case 13:
|
|
159858
|
+
internal::__fastunpack13(in, out);
|
|
159859
|
+
break;
|
|
159860
|
+
case 14:
|
|
159861
|
+
internal::__fastunpack14(in, out);
|
|
159862
|
+
break;
|
|
159863
|
+
case 15:
|
|
159864
|
+
internal::__fastunpack15(in, out);
|
|
159865
|
+
break;
|
|
159866
|
+
case 16:
|
|
159867
|
+
internal::__fastunpack16(in, out);
|
|
159868
|
+
break;
|
|
159869
|
+
case 17:
|
|
159870
|
+
internal::__fastunpack17(in, out);
|
|
159871
|
+
break;
|
|
159872
|
+
case 18:
|
|
159873
|
+
internal::__fastunpack18(in, out);
|
|
159874
|
+
break;
|
|
159875
|
+
case 19:
|
|
159876
|
+
internal::__fastunpack19(in, out);
|
|
159877
|
+
break;
|
|
159878
|
+
case 20:
|
|
159879
|
+
internal::__fastunpack20(in, out);
|
|
159880
|
+
break;
|
|
159881
|
+
case 21:
|
|
159882
|
+
internal::__fastunpack21(in, out);
|
|
159883
|
+
break;
|
|
159884
|
+
case 22:
|
|
159885
|
+
internal::__fastunpack22(in, out);
|
|
159886
|
+
break;
|
|
159887
|
+
case 23:
|
|
159888
|
+
internal::__fastunpack23(in, out);
|
|
159889
|
+
break;
|
|
159890
|
+
case 24:
|
|
159891
|
+
internal::__fastunpack24(in, out);
|
|
159892
|
+
break;
|
|
159893
|
+
case 25:
|
|
159894
|
+
internal::__fastunpack25(in, out);
|
|
159895
|
+
break;
|
|
159896
|
+
case 26:
|
|
159897
|
+
internal::__fastunpack26(in, out);
|
|
159898
|
+
break;
|
|
159899
|
+
case 27:
|
|
159900
|
+
internal::__fastunpack27(in, out);
|
|
159901
|
+
break;
|
|
159902
|
+
case 28:
|
|
159903
|
+
internal::__fastunpack28(in, out);
|
|
159904
|
+
break;
|
|
159905
|
+
case 29:
|
|
159906
|
+
internal::__fastunpack29(in, out);
|
|
159907
|
+
break;
|
|
159908
|
+
case 30:
|
|
159909
|
+
internal::__fastunpack30(in, out);
|
|
159910
|
+
break;
|
|
159911
|
+
case 31:
|
|
159912
|
+
internal::__fastunpack31(in, out);
|
|
159913
|
+
break;
|
|
159914
|
+
case 32:
|
|
159915
|
+
internal::__fastunpack32(in, out);
|
|
159916
|
+
break;
|
|
159917
|
+
default:
|
|
159918
|
+
throw std::logic_error("Invalid bit width for bitpacking");
|
|
159919
|
+
}
|
|
159920
|
+
}
|
|
159921
|
+
|
|
159922
|
+
inline void fastunpack(const uint32_t *__restrict in,
|
|
159923
|
+
uint64_t *__restrict out, const uint32_t bit) {
|
|
159924
|
+
// Could have used function pointers instead of switch.
|
|
159925
|
+
// Switch calls do offer the compiler more opportunities for optimization in
|
|
159926
|
+
// theory. In this case, it makes no difference with a good compiler.
|
|
159927
|
+
switch (bit) {
|
|
159928
|
+
case 0:
|
|
159929
|
+
internal::__fastunpack0(in, out);
|
|
159930
|
+
break;
|
|
159931
|
+
case 1:
|
|
159932
|
+
internal::__fastunpack1(in, out);
|
|
159933
|
+
break;
|
|
159934
|
+
case 2:
|
|
159935
|
+
internal::__fastunpack2(in, out);
|
|
159936
|
+
break;
|
|
159937
|
+
case 3:
|
|
159938
|
+
internal::__fastunpack3(in, out);
|
|
159939
|
+
break;
|
|
159940
|
+
case 4:
|
|
159941
|
+
internal::__fastunpack4(in, out);
|
|
159942
|
+
break;
|
|
159943
|
+
case 5:
|
|
159944
|
+
internal::__fastunpack5(in, out);
|
|
159945
|
+
break;
|
|
159946
|
+
case 6:
|
|
159947
|
+
internal::__fastunpack6(in, out);
|
|
159948
|
+
break;
|
|
159949
|
+
case 7:
|
|
159950
|
+
internal::__fastunpack7(in, out);
|
|
159951
|
+
break;
|
|
159952
|
+
case 8:
|
|
159953
|
+
internal::__fastunpack8(in, out);
|
|
159954
|
+
break;
|
|
159955
|
+
case 9:
|
|
159956
|
+
internal::__fastunpack9(in, out);
|
|
159957
|
+
break;
|
|
159958
|
+
case 10:
|
|
159959
|
+
internal::__fastunpack10(in, out);
|
|
159960
|
+
break;
|
|
159961
|
+
case 11:
|
|
159962
|
+
internal::__fastunpack11(in, out);
|
|
159963
|
+
break;
|
|
159964
|
+
case 12:
|
|
159965
|
+
internal::__fastunpack12(in, out);
|
|
159966
|
+
break;
|
|
159967
|
+
case 13:
|
|
159968
|
+
internal::__fastunpack13(in, out);
|
|
159969
|
+
break;
|
|
159970
|
+
case 14:
|
|
159971
|
+
internal::__fastunpack14(in, out);
|
|
159972
|
+
break;
|
|
159973
|
+
case 15:
|
|
159974
|
+
internal::__fastunpack15(in, out);
|
|
159975
|
+
break;
|
|
159976
|
+
case 16:
|
|
159977
|
+
internal::__fastunpack16(in, out);
|
|
159978
|
+
break;
|
|
159979
|
+
case 17:
|
|
159980
|
+
internal::__fastunpack17(in, out);
|
|
159981
|
+
break;
|
|
159982
|
+
case 18:
|
|
159983
|
+
internal::__fastunpack18(in, out);
|
|
159984
|
+
break;
|
|
159985
|
+
case 19:
|
|
159986
|
+
internal::__fastunpack19(in, out);
|
|
159987
|
+
break;
|
|
159988
|
+
case 20:
|
|
159989
|
+
internal::__fastunpack20(in, out);
|
|
159990
|
+
break;
|
|
159991
|
+
case 21:
|
|
159992
|
+
internal::__fastunpack21(in, out);
|
|
159993
|
+
break;
|
|
159994
|
+
case 22:
|
|
159995
|
+
internal::__fastunpack22(in, out);
|
|
159996
|
+
break;
|
|
159997
|
+
case 23:
|
|
159998
|
+
internal::__fastunpack23(in, out);
|
|
159999
|
+
break;
|
|
160000
|
+
case 24:
|
|
160001
|
+
internal::__fastunpack24(in, out);
|
|
160002
|
+
break;
|
|
160003
|
+
case 25:
|
|
160004
|
+
internal::__fastunpack25(in, out);
|
|
160005
|
+
break;
|
|
160006
|
+
case 26:
|
|
160007
|
+
internal::__fastunpack26(in, out);
|
|
160008
|
+
break;
|
|
160009
|
+
case 27:
|
|
160010
|
+
internal::__fastunpack27(in, out);
|
|
160011
|
+
break;
|
|
160012
|
+
case 28:
|
|
160013
|
+
internal::__fastunpack28(in, out);
|
|
160014
|
+
break;
|
|
160015
|
+
case 29:
|
|
160016
|
+
internal::__fastunpack29(in, out);
|
|
160017
|
+
break;
|
|
160018
|
+
case 30:
|
|
160019
|
+
internal::__fastunpack30(in, out);
|
|
160020
|
+
break;
|
|
160021
|
+
case 31:
|
|
160022
|
+
internal::__fastunpack31(in, out);
|
|
160023
|
+
break;
|
|
160024
|
+
case 32:
|
|
160025
|
+
internal::__fastunpack32(in, out);
|
|
160026
|
+
break;
|
|
160027
|
+
case 33:
|
|
160028
|
+
internal::__fastunpack33(in, out);
|
|
160029
|
+
break;
|
|
160030
|
+
case 34:
|
|
160031
|
+
internal::__fastunpack34(in, out);
|
|
160032
|
+
break;
|
|
160033
|
+
case 35:
|
|
160034
|
+
internal::__fastunpack35(in, out);
|
|
160035
|
+
break;
|
|
160036
|
+
case 36:
|
|
160037
|
+
internal::__fastunpack36(in, out);
|
|
160038
|
+
break;
|
|
160039
|
+
case 37:
|
|
160040
|
+
internal::__fastunpack37(in, out);
|
|
160041
|
+
break;
|
|
160042
|
+
case 38:
|
|
160043
|
+
internal::__fastunpack38(in, out);
|
|
160044
|
+
break;
|
|
160045
|
+
case 39:
|
|
160046
|
+
internal::__fastunpack39(in, out);
|
|
160047
|
+
break;
|
|
160048
|
+
case 40:
|
|
160049
|
+
internal::__fastunpack40(in, out);
|
|
160050
|
+
break;
|
|
160051
|
+
case 41:
|
|
160052
|
+
internal::__fastunpack41(in, out);
|
|
160053
|
+
break;
|
|
160054
|
+
case 42:
|
|
160055
|
+
internal::__fastunpack42(in, out);
|
|
160056
|
+
break;
|
|
160057
|
+
case 43:
|
|
160058
|
+
internal::__fastunpack43(in, out);
|
|
160059
|
+
break;
|
|
160060
|
+
case 44:
|
|
160061
|
+
internal::__fastunpack44(in, out);
|
|
160062
|
+
break;
|
|
160063
|
+
case 45:
|
|
160064
|
+
internal::__fastunpack45(in, out);
|
|
160065
|
+
break;
|
|
160066
|
+
case 46:
|
|
160067
|
+
internal::__fastunpack46(in, out);
|
|
160068
|
+
break;
|
|
160069
|
+
case 47:
|
|
160070
|
+
internal::__fastunpack47(in, out);
|
|
160071
|
+
break;
|
|
160072
|
+
case 48:
|
|
160073
|
+
internal::__fastunpack48(in, out);
|
|
160074
|
+
break;
|
|
160075
|
+
case 49:
|
|
160076
|
+
internal::__fastunpack49(in, out);
|
|
160077
|
+
break;
|
|
160078
|
+
case 50:
|
|
160079
|
+
internal::__fastunpack50(in, out);
|
|
160080
|
+
break;
|
|
160081
|
+
case 51:
|
|
160082
|
+
internal::__fastunpack51(in, out);
|
|
160083
|
+
break;
|
|
160084
|
+
case 52:
|
|
160085
|
+
internal::__fastunpack52(in, out);
|
|
160086
|
+
break;
|
|
160087
|
+
case 53:
|
|
160088
|
+
internal::__fastunpack53(in, out);
|
|
160089
|
+
break;
|
|
160090
|
+
case 54:
|
|
160091
|
+
internal::__fastunpack54(in, out);
|
|
160092
|
+
break;
|
|
160093
|
+
case 55:
|
|
160094
|
+
internal::__fastunpack55(in, out);
|
|
160095
|
+
break;
|
|
160096
|
+
case 56:
|
|
160097
|
+
internal::__fastunpack56(in, out);
|
|
160098
|
+
break;
|
|
160099
|
+
case 57:
|
|
160100
|
+
internal::__fastunpack57(in, out);
|
|
160101
|
+
break;
|
|
160102
|
+
case 58:
|
|
160103
|
+
internal::__fastunpack58(in, out);
|
|
160104
|
+
break;
|
|
160105
|
+
case 59:
|
|
160106
|
+
internal::__fastunpack59(in, out);
|
|
160107
|
+
break;
|
|
160108
|
+
case 60:
|
|
160109
|
+
internal::__fastunpack60(in, out);
|
|
160110
|
+
break;
|
|
160111
|
+
case 61:
|
|
160112
|
+
internal::__fastunpack61(in, out);
|
|
160113
|
+
break;
|
|
160114
|
+
case 62:
|
|
160115
|
+
internal::__fastunpack62(in, out);
|
|
160116
|
+
break;
|
|
160117
|
+
case 63:
|
|
160118
|
+
internal::__fastunpack63(in, out);
|
|
160119
|
+
break;
|
|
160120
|
+
case 64:
|
|
160121
|
+
internal::__fastunpack64(in, out);
|
|
160122
|
+
break;
|
|
160123
|
+
default:
|
|
160124
|
+
throw std::logic_error("Invalid bit width for bitpacking");
|
|
160125
|
+
}
|
|
160126
|
+
}
|
|
160127
|
+
|
|
160128
|
+
inline void fastpack(const uint8_t *__restrict in, uint8_t *__restrict out, const uint32_t bit) {
|
|
160129
|
+
|
|
160130
|
+
for (uint8_t i = 0; i < 4; i++) {
|
|
160131
|
+
internal::fastpack_quarter(in+(i*8), out + (i*bit), bit);
|
|
160132
|
+
}
|
|
160133
|
+
}
|
|
160134
|
+
|
|
160135
|
+
inline void fastpack(const uint16_t *__restrict in, uint16_t *__restrict out, const uint32_t bit) {
|
|
160136
|
+
internal::fastpack_half(in, out, bit);
|
|
160137
|
+
internal::fastpack_half(in+16, out + bit, bit);
|
|
160138
|
+
}
|
|
160139
|
+
|
|
160140
|
+
inline void fastpack(const uint32_t *__restrict in,
|
|
160141
|
+
uint32_t *__restrict out, const uint32_t bit) {
|
|
160142
|
+
// Could have used function pointers instead of switch.
|
|
160143
|
+
// Switch calls do offer the compiler more opportunities for optimization in
|
|
160144
|
+
// theory. In this case, it makes no difference with a good compiler.
|
|
160145
|
+
switch (bit) {
|
|
160146
|
+
case 0:
|
|
160147
|
+
internal::__fastpack0(in, out);
|
|
160148
|
+
break;
|
|
160149
|
+
case 1:
|
|
160150
|
+
internal::__fastpack1(in, out);
|
|
160151
|
+
break;
|
|
160152
|
+
case 2:
|
|
160153
|
+
internal::__fastpack2(in, out);
|
|
160154
|
+
break;
|
|
160155
|
+
case 3:
|
|
160156
|
+
internal::__fastpack3(in, out);
|
|
160157
|
+
break;
|
|
160158
|
+
case 4:
|
|
160159
|
+
internal::__fastpack4(in, out);
|
|
160160
|
+
break;
|
|
160161
|
+
case 5:
|
|
160162
|
+
internal::__fastpack5(in, out);
|
|
160163
|
+
break;
|
|
160164
|
+
case 6:
|
|
160165
|
+
internal::__fastpack6(in, out);
|
|
160166
|
+
break;
|
|
160167
|
+
case 7:
|
|
160168
|
+
internal::__fastpack7(in, out);
|
|
160169
|
+
break;
|
|
160170
|
+
case 8:
|
|
160171
|
+
internal::__fastpack8(in, out);
|
|
160172
|
+
break;
|
|
160173
|
+
case 9:
|
|
160174
|
+
internal::__fastpack9(in, out);
|
|
160175
|
+
break;
|
|
160176
|
+
case 10:
|
|
160177
|
+
internal::__fastpack10(in, out);
|
|
160178
|
+
break;
|
|
160179
|
+
case 11:
|
|
160180
|
+
internal::__fastpack11(in, out);
|
|
160181
|
+
break;
|
|
160182
|
+
case 12:
|
|
160183
|
+
internal::__fastpack12(in, out);
|
|
160184
|
+
break;
|
|
160185
|
+
case 13:
|
|
160186
|
+
internal::__fastpack13(in, out);
|
|
160187
|
+
break;
|
|
160188
|
+
case 14:
|
|
160189
|
+
internal::__fastpack14(in, out);
|
|
160190
|
+
break;
|
|
160191
|
+
case 15:
|
|
160192
|
+
internal::__fastpack15(in, out);
|
|
160193
|
+
break;
|
|
160194
|
+
case 16:
|
|
160195
|
+
internal::__fastpack16(in, out);
|
|
160196
|
+
break;
|
|
160197
|
+
case 17:
|
|
160198
|
+
internal::__fastpack17(in, out);
|
|
160199
|
+
break;
|
|
160200
|
+
case 18:
|
|
160201
|
+
internal::__fastpack18(in, out);
|
|
160202
|
+
break;
|
|
160203
|
+
case 19:
|
|
160204
|
+
internal::__fastpack19(in, out);
|
|
160205
|
+
break;
|
|
160206
|
+
case 20:
|
|
160207
|
+
internal::__fastpack20(in, out);
|
|
160208
|
+
break;
|
|
160209
|
+
case 21:
|
|
160210
|
+
internal::__fastpack21(in, out);
|
|
160211
|
+
break;
|
|
160212
|
+
case 22:
|
|
160213
|
+
internal::__fastpack22(in, out);
|
|
160214
|
+
break;
|
|
160215
|
+
case 23:
|
|
160216
|
+
internal::__fastpack23(in, out);
|
|
160217
|
+
break;
|
|
160218
|
+
case 24:
|
|
160219
|
+
internal::__fastpack24(in, out);
|
|
160220
|
+
break;
|
|
160221
|
+
case 25:
|
|
160222
|
+
internal::__fastpack25(in, out);
|
|
160223
|
+
break;
|
|
160224
|
+
case 26:
|
|
160225
|
+
internal::__fastpack26(in, out);
|
|
160226
|
+
break;
|
|
160227
|
+
case 27:
|
|
160228
|
+
internal::__fastpack27(in, out);
|
|
160229
|
+
break;
|
|
160230
|
+
case 28:
|
|
160231
|
+
internal::__fastpack28(in, out);
|
|
160232
|
+
break;
|
|
160233
|
+
case 29:
|
|
160234
|
+
internal::__fastpack29(in, out);
|
|
160235
|
+
break;
|
|
160236
|
+
case 30:
|
|
160237
|
+
internal::__fastpack30(in, out);
|
|
160238
|
+
break;
|
|
160239
|
+
case 31:
|
|
160240
|
+
internal::__fastpack31(in, out);
|
|
160241
|
+
break;
|
|
160242
|
+
case 32:
|
|
160243
|
+
internal::__fastpack32(in, out);
|
|
160244
|
+
break;
|
|
160245
|
+
default:
|
|
160246
|
+
throw std::logic_error("Invalid bit width for bitpacking");
|
|
160247
|
+
}
|
|
160248
|
+
}
|
|
160249
|
+
|
|
160250
|
+
inline void fastpack(const uint64_t *__restrict in,
|
|
160251
|
+
uint32_t *__restrict out, const uint32_t bit) {
|
|
160252
|
+
switch (bit) {
|
|
160253
|
+
case 0:
|
|
160254
|
+
internal::__fastpack0(in, out);
|
|
160255
|
+
break;
|
|
160256
|
+
case 1:
|
|
160257
|
+
internal::__fastpack1(in, out);
|
|
160258
|
+
break;
|
|
160259
|
+
case 2:
|
|
160260
|
+
internal::__fastpack2(in, out);
|
|
160261
|
+
break;
|
|
160262
|
+
case 3:
|
|
160263
|
+
internal::__fastpack3(in, out);
|
|
160264
|
+
break;
|
|
160265
|
+
case 4:
|
|
160266
|
+
internal::__fastpack4(in, out);
|
|
160267
|
+
break;
|
|
160268
|
+
case 5:
|
|
160269
|
+
internal::__fastpack5(in, out);
|
|
160270
|
+
break;
|
|
160271
|
+
case 6:
|
|
160272
|
+
internal::__fastpack6(in, out);
|
|
160273
|
+
break;
|
|
160274
|
+
case 7:
|
|
160275
|
+
internal::__fastpack7(in, out);
|
|
160276
|
+
break;
|
|
160277
|
+
case 8:
|
|
160278
|
+
internal::__fastpack8(in, out);
|
|
160279
|
+
break;
|
|
160280
|
+
case 9:
|
|
160281
|
+
internal::__fastpack9(in, out);
|
|
160282
|
+
break;
|
|
160283
|
+
case 10:
|
|
160284
|
+
internal::__fastpack10(in, out);
|
|
160285
|
+
break;
|
|
160286
|
+
case 11:
|
|
160287
|
+
internal::__fastpack11(in, out);
|
|
160288
|
+
break;
|
|
160289
|
+
case 12:
|
|
160290
|
+
internal::__fastpack12(in, out);
|
|
160291
|
+
break;
|
|
160292
|
+
case 13:
|
|
160293
|
+
internal::__fastpack13(in, out);
|
|
160294
|
+
break;
|
|
160295
|
+
case 14:
|
|
160296
|
+
internal::__fastpack14(in, out);
|
|
160297
|
+
break;
|
|
160298
|
+
case 15:
|
|
160299
|
+
internal::__fastpack15(in, out);
|
|
160300
|
+
break;
|
|
160301
|
+
case 16:
|
|
160302
|
+
internal::__fastpack16(in, out);
|
|
160303
|
+
break;
|
|
160304
|
+
case 17:
|
|
160305
|
+
internal::__fastpack17(in, out);
|
|
160306
|
+
break;
|
|
160307
|
+
case 18:
|
|
160308
|
+
internal::__fastpack18(in, out);
|
|
160309
|
+
break;
|
|
160310
|
+
case 19:
|
|
160311
|
+
internal::__fastpack19(in, out);
|
|
160312
|
+
break;
|
|
160313
|
+
case 20:
|
|
160314
|
+
internal::__fastpack20(in, out);
|
|
160315
|
+
break;
|
|
160316
|
+
case 21:
|
|
160317
|
+
internal::__fastpack21(in, out);
|
|
160318
|
+
break;
|
|
160319
|
+
case 22:
|
|
160320
|
+
internal::__fastpack22(in, out);
|
|
160321
|
+
break;
|
|
160322
|
+
case 23:
|
|
160323
|
+
internal::__fastpack23(in, out);
|
|
160324
|
+
break;
|
|
160325
|
+
case 24:
|
|
160326
|
+
internal::__fastpack24(in, out);
|
|
160327
|
+
break;
|
|
160328
|
+
case 25:
|
|
160329
|
+
internal::__fastpack25(in, out);
|
|
160330
|
+
break;
|
|
160331
|
+
case 26:
|
|
160332
|
+
internal::__fastpack26(in, out);
|
|
160333
|
+
break;
|
|
160334
|
+
case 27:
|
|
160335
|
+
internal::__fastpack27(in, out);
|
|
160336
|
+
break;
|
|
160337
|
+
case 28:
|
|
160338
|
+
internal::__fastpack28(in, out);
|
|
160339
|
+
break;
|
|
160340
|
+
case 29:
|
|
160341
|
+
internal::__fastpack29(in, out);
|
|
160342
|
+
break;
|
|
160343
|
+
case 30:
|
|
160344
|
+
internal::__fastpack30(in, out);
|
|
160345
|
+
break;
|
|
160346
|
+
case 31:
|
|
160347
|
+
internal::__fastpack31(in, out);
|
|
160348
|
+
break;
|
|
160349
|
+
case 32:
|
|
160350
|
+
internal::__fastpack32(in, out);
|
|
160351
|
+
break;
|
|
160352
|
+
case 33:
|
|
160353
|
+
internal::__fastpack33(in, out);
|
|
160354
|
+
break;
|
|
160355
|
+
case 34:
|
|
160356
|
+
internal::__fastpack34(in, out);
|
|
160357
|
+
break;
|
|
160358
|
+
case 35:
|
|
160359
|
+
internal::__fastpack35(in, out);
|
|
160360
|
+
break;
|
|
160361
|
+
case 36:
|
|
160362
|
+
internal::__fastpack36(in, out);
|
|
160363
|
+
break;
|
|
160364
|
+
case 37:
|
|
160365
|
+
internal::__fastpack37(in, out);
|
|
160366
|
+
break;
|
|
160367
|
+
case 38:
|
|
160368
|
+
internal::__fastpack38(in, out);
|
|
160369
|
+
break;
|
|
160370
|
+
case 39:
|
|
160371
|
+
internal::__fastpack39(in, out);
|
|
160372
|
+
break;
|
|
160373
|
+
case 40:
|
|
160374
|
+
internal::__fastpack40(in, out);
|
|
160375
|
+
break;
|
|
160376
|
+
case 41:
|
|
160377
|
+
internal::__fastpack41(in, out);
|
|
160378
|
+
break;
|
|
160379
|
+
case 42:
|
|
160380
|
+
internal::__fastpack42(in, out);
|
|
160381
|
+
break;
|
|
160382
|
+
case 43:
|
|
160383
|
+
internal::__fastpack43(in, out);
|
|
160384
|
+
break;
|
|
160385
|
+
case 44:
|
|
160386
|
+
internal::__fastpack44(in, out);
|
|
160387
|
+
break;
|
|
160388
|
+
case 45:
|
|
160389
|
+
internal::__fastpack45(in, out);
|
|
160390
|
+
break;
|
|
160391
|
+
case 46:
|
|
160392
|
+
internal::__fastpack46(in, out);
|
|
160393
|
+
break;
|
|
160394
|
+
case 47:
|
|
160395
|
+
internal::__fastpack47(in, out);
|
|
160396
|
+
break;
|
|
160397
|
+
case 48:
|
|
160398
|
+
internal::__fastpack48(in, out);
|
|
160399
|
+
break;
|
|
160400
|
+
case 49:
|
|
160401
|
+
internal::__fastpack49(in, out);
|
|
160402
|
+
break;
|
|
160403
|
+
case 50:
|
|
160404
|
+
internal::__fastpack50(in, out);
|
|
160405
|
+
break;
|
|
160406
|
+
case 51:
|
|
160407
|
+
internal::__fastpack51(in, out);
|
|
160408
|
+
break;
|
|
160409
|
+
case 52:
|
|
160410
|
+
internal::__fastpack52(in, out);
|
|
160411
|
+
break;
|
|
160412
|
+
case 53:
|
|
160413
|
+
internal::__fastpack53(in, out);
|
|
160414
|
+
break;
|
|
160415
|
+
case 54:
|
|
160416
|
+
internal::__fastpack54(in, out);
|
|
160417
|
+
break;
|
|
160418
|
+
case 55:
|
|
160419
|
+
internal::__fastpack55(in, out);
|
|
160420
|
+
break;
|
|
160421
|
+
case 56:
|
|
160422
|
+
internal::__fastpack56(in, out);
|
|
160423
|
+
break;
|
|
160424
|
+
case 57:
|
|
160425
|
+
internal::__fastpack57(in, out);
|
|
160426
|
+
break;
|
|
160427
|
+
case 58:
|
|
160428
|
+
internal::__fastpack58(in, out);
|
|
160429
|
+
break;
|
|
160430
|
+
case 59:
|
|
160431
|
+
internal::__fastpack59(in, out);
|
|
160432
|
+
break;
|
|
160433
|
+
case 60:
|
|
160434
|
+
internal::__fastpack60(in, out);
|
|
160435
|
+
break;
|
|
160436
|
+
case 61:
|
|
160437
|
+
internal::__fastpack61(in, out);
|
|
160438
|
+
break;
|
|
160439
|
+
case 62:
|
|
160440
|
+
internal::__fastpack62(in, out);
|
|
160441
|
+
break;
|
|
160442
|
+
case 63:
|
|
160443
|
+
internal::__fastpack63(in, out);
|
|
160444
|
+
break;
|
|
160445
|
+
case 64:
|
|
160446
|
+
internal::__fastpack64(in, out);
|
|
160447
|
+
break;
|
|
160448
|
+
default:
|
|
160449
|
+
throw std::logic_error("Invalid bit width for bitpacking");
|
|
160450
|
+
}
|
|
160451
|
+
}
|
|
160452
|
+
} // namespace fastpfor_lib
|
|
159333
160453
|
|
|
159334
160454
|
|
|
160455
|
+
// LICENSE_CHANGE_END
|
|
159335
160456
|
|
|
159336
160457
|
|
|
159337
160458
|
|
|
@@ -159578,6 +160699,691 @@ private:
|
|
|
159578
160699
|
} // namespace duckdb
|
|
159579
160700
|
|
|
159580
160701
|
|
|
160702
|
+
|
|
160703
|
+
#include <functional>
|
|
160704
|
+
|
|
160705
|
+
namespace duckdb {
|
|
160706
|
+
|
|
160707
|
+
using bitpacking_width_t = uint8_t;
|
|
160708
|
+
|
|
160709
|
+
// Note that optimizations in scanning only work if this value is equal to STANDARD_VECTOR_SIZE, however we keep them
|
|
160710
|
+
// separated to prevent the code from break on lower vector sizes
|
|
160711
|
+
static constexpr const idx_t BITPACKING_WIDTH_GROUP_SIZE = 1024;
|
|
160712
|
+
|
|
160713
|
+
class BitpackingPrimitives {
|
|
160714
|
+
public:
|
|
160715
|
+
static constexpr const idx_t BITPACKING_ALGORITHM_GROUP_SIZE = 32;
|
|
160716
|
+
static constexpr const idx_t BITPACKING_HEADER_SIZE = sizeof(uint64_t);
|
|
160717
|
+
static constexpr const bool BYTE_ALIGNED = false;
|
|
160718
|
+
|
|
160719
|
+
// Packs a block of BITPACKING_ALGORITHM_GROUP_SIZE values
|
|
160720
|
+
template <class T>
|
|
160721
|
+
inline static void PackBlock(data_ptr_t dst, T *src, bitpacking_width_t width) {
|
|
160722
|
+
return PackGroup<T>(dst, src, width);
|
|
160723
|
+
}
|
|
160724
|
+
|
|
160725
|
+
// Unpacks a block of BITPACKING_ALGORITHM_GROUP_SIZE values
|
|
160726
|
+
template <class T>
|
|
160727
|
+
inline static void UnPackBlock(data_ptr_t dst, data_ptr_t src, bitpacking_width_t width,
|
|
160728
|
+
bool skip_sign_extension = false) {
|
|
160729
|
+
return UnPackGroup<T>(dst, src, width, skip_sign_extension);
|
|
160730
|
+
}
|
|
160731
|
+
|
|
160732
|
+
// Calculates the minimum required number of bits per value that can store all values
|
|
160733
|
+
template <class T>
|
|
160734
|
+
inline static bitpacking_width_t MinimumBitWidth(T *values, idx_t count) {
|
|
160735
|
+
return FindMinimumBitWidth<T, BYTE_ALIGNED>(values, count);
|
|
160736
|
+
}
|
|
160737
|
+
|
|
160738
|
+
private:
|
|
160739
|
+
template <class T>
|
|
160740
|
+
static bitpacking_width_t MinimumBitWidth(T min_value, T max_value) {
|
|
160741
|
+
bitpacking_width_t required_bits;
|
|
160742
|
+
|
|
160743
|
+
if (std::is_signed<T>::value) {
|
|
160744
|
+
if (min_value == NumericLimits<T>::Minimum()) {
|
|
160745
|
+
// handle special case of the minimal value, as it cannot be negated like all other values.
|
|
160746
|
+
return sizeof(T) * 8;
|
|
160747
|
+
} else {
|
|
160748
|
+
max_value = MaxValue((T)-min_value, max_value);
|
|
160749
|
+
}
|
|
160750
|
+
}
|
|
160751
|
+
|
|
160752
|
+
if (max_value == 0) {
|
|
160753
|
+
return 0;
|
|
160754
|
+
}
|
|
160755
|
+
|
|
160756
|
+
if (std::is_signed<T>::value) {
|
|
160757
|
+
required_bits = 1;
|
|
160758
|
+
} else {
|
|
160759
|
+
required_bits = 0;
|
|
160760
|
+
}
|
|
160761
|
+
|
|
160762
|
+
while (max_value) {
|
|
160763
|
+
required_bits++;
|
|
160764
|
+
max_value >>= 1;
|
|
160765
|
+
}
|
|
160766
|
+
|
|
160767
|
+
return GetEffectiveWidth<T>(required_bits);
|
|
160768
|
+
}
|
|
160769
|
+
|
|
160770
|
+
template <class T, bool round_to_next_byte = false>
|
|
160771
|
+
static bitpacking_width_t FindMinimumBitWidth(T *values, idx_t count) {
|
|
160772
|
+
T min_value = values[0];
|
|
160773
|
+
T max_value = values[0];
|
|
160774
|
+
|
|
160775
|
+
for (idx_t i = 1; i < count; i++) {
|
|
160776
|
+
if (values[i] > max_value) {
|
|
160777
|
+
max_value = values[i];
|
|
160778
|
+
}
|
|
160779
|
+
|
|
160780
|
+
if (std::is_signed<T>::value) {
|
|
160781
|
+
if (values[i] < min_value) {
|
|
160782
|
+
min_value = values[i];
|
|
160783
|
+
}
|
|
160784
|
+
}
|
|
160785
|
+
}
|
|
160786
|
+
|
|
160787
|
+
bitpacking_width_t calc_width = MinimumBitWidth<T>(std::is_signed<T>::value ? min_value : 0, max_value);
|
|
160788
|
+
|
|
160789
|
+
// Assert results are correct
|
|
160790
|
+
#ifdef DEBUG
|
|
160791
|
+
if (calc_width < sizeof(T) * 8 && calc_width != 0) {
|
|
160792
|
+
if (std::is_signed<T>::value) {
|
|
160793
|
+
D_ASSERT((int64_t)max_value <= (int64_t)(1L << (calc_width - 1)) - 1);
|
|
160794
|
+
D_ASSERT((int64_t)min_value >= (int64_t)(-1 * ((1L << (calc_width - 1)) - 1) - 1));
|
|
160795
|
+
} else {
|
|
160796
|
+
D_ASSERT((uint64_t)max_value <= (uint64_t)(1L << (calc_width)) - 1);
|
|
160797
|
+
}
|
|
160798
|
+
}
|
|
160799
|
+
#endif
|
|
160800
|
+
if (round_to_next_byte) {
|
|
160801
|
+
return (calc_width / 8 + (calc_width % 8 != 0)) * 8;
|
|
160802
|
+
} else {
|
|
160803
|
+
return calc_width;
|
|
160804
|
+
}
|
|
160805
|
+
}
|
|
160806
|
+
|
|
160807
|
+
template <class T>
|
|
160808
|
+
static void UnPackGroup(data_ptr_t dst, data_ptr_t src, bitpacking_width_t width,
|
|
160809
|
+
bool skip_sign_extension = false) {
|
|
160810
|
+
if (std::is_same<T, uint8_t>::value || std::is_same<T, int8_t>::value) {
|
|
160811
|
+
duckdb_fastpforlib::fastunpack((const uint8_t *)src, (uint8_t *)dst, (uint32_t)width);
|
|
160812
|
+
} else if (std::is_same<T, uint16_t>::value || std::is_same<T, int16_t>::value) {
|
|
160813
|
+
duckdb_fastpforlib::fastunpack((const uint16_t *)src, (uint16_t *)dst, (uint32_t)width);
|
|
160814
|
+
} else if (std::is_same<T, uint32_t>::value || std::is_same<T, int32_t>::value) {
|
|
160815
|
+
duckdb_fastpforlib::fastunpack((const uint32_t *)src, (uint32_t *)dst, (uint32_t)width);
|
|
160816
|
+
} else if (std::is_same<T, uint64_t>::value || std::is_same<T, int64_t>::value) {
|
|
160817
|
+
duckdb_fastpforlib::fastunpack((const uint32_t *)src, (uint64_t *)dst, (uint32_t)width);
|
|
160818
|
+
} else {
|
|
160819
|
+
throw InternalException("Unsupported type found in bitpacking.");
|
|
160820
|
+
}
|
|
160821
|
+
|
|
160822
|
+
if (NumericLimits<T>::IsSigned() && !skip_sign_extension && width > 0 && width < sizeof(T) * 8) {
|
|
160823
|
+
SignExtend<T>(dst, width);
|
|
160824
|
+
}
|
|
160825
|
+
}
|
|
160826
|
+
|
|
160827
|
+
// Prevent compression at widths that are ineffective
|
|
160828
|
+
template <class T>
|
|
160829
|
+
static bitpacking_width_t GetEffectiveWidth(bitpacking_width_t width) {
|
|
160830
|
+
if (width > 56) {
|
|
160831
|
+
return 64;
|
|
160832
|
+
}
|
|
160833
|
+
|
|
160834
|
+
if (width > 28 && (std::is_same<T, uint32_t>::value || std::is_same<T, int32_t>::value)) {
|
|
160835
|
+
return 32;
|
|
160836
|
+
}
|
|
160837
|
+
|
|
160838
|
+
else if (width > 14 && (std::is_same<T, uint16_t>::value || std::is_same<T, int16_t>::value)) {
|
|
160839
|
+
return 16;
|
|
160840
|
+
}
|
|
160841
|
+
|
|
160842
|
+
return width;
|
|
160843
|
+
}
|
|
160844
|
+
|
|
160845
|
+
// Sign bit extension
|
|
160846
|
+
template <class T, class T_U = typename std::make_unsigned<T>::type>
|
|
160847
|
+
static void SignExtend(data_ptr_t dst, bitpacking_width_t width) {
|
|
160848
|
+
T const mask = ((T_U)1) << (width - 1);
|
|
160849
|
+
for (idx_t i = 0; i < BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE; ++i) {
|
|
160850
|
+
T value = Load<T>(dst + i * sizeof(T));
|
|
160851
|
+
value = value & ((((T_U)1) << width) - ((T_U)1));
|
|
160852
|
+
T result = (value ^ mask) - mask;
|
|
160853
|
+
Store(result, dst + i * sizeof(T));
|
|
160854
|
+
}
|
|
160855
|
+
}
|
|
160856
|
+
|
|
160857
|
+
template <class T>
|
|
160858
|
+
static void PackGroup(data_ptr_t dst, T *values, bitpacking_width_t width) {
|
|
160859
|
+
if (std::is_same<T, uint8_t>::value || std::is_same<T, int8_t>::value) {
|
|
160860
|
+
duckdb_fastpforlib::fastpack((const uint8_t *)values, (uint8_t *)dst, (uint32_t)width);
|
|
160861
|
+
} else if (std::is_same<T, uint16_t>::value || std::is_same<T, int16_t>::value) {
|
|
160862
|
+
duckdb_fastpforlib::fastpack((const uint16_t *)values, (uint16_t *)dst, (uint32_t)width);
|
|
160863
|
+
} else if (std::is_same<T, uint32_t>::value || std::is_same<T, int32_t>::value) {
|
|
160864
|
+
duckdb_fastpforlib::fastpack((const uint32_t *)values, (uint32_t *)dst, (uint32_t)width);
|
|
160865
|
+
} else if (std::is_same<T, uint64_t>::value || std::is_same<T, int64_t>::value) {
|
|
160866
|
+
duckdb_fastpforlib::fastpack((const uint64_t *)values, (uint32_t *)dst, (uint32_t)width);
|
|
160867
|
+
} else {
|
|
160868
|
+
throw InternalException("Unsupported type found in bitpacking.");
|
|
160869
|
+
}
|
|
160870
|
+
}
|
|
160871
|
+
};
|
|
160872
|
+
|
|
160873
|
+
struct EmptyBitpackingWriter {
|
|
160874
|
+
template <class T>
|
|
160875
|
+
static void Operation(T *values, bool *validity, bitpacking_width_t width, idx_t count, void *data_ptr) {
|
|
160876
|
+
}
|
|
160877
|
+
};
|
|
160878
|
+
|
|
160879
|
+
template <class T>
|
|
160880
|
+
struct BitpackingState {
|
|
160881
|
+
public:
|
|
160882
|
+
BitpackingState() : compression_buffer_idx(0), total_size(0), data_ptr(nullptr) {
|
|
160883
|
+
}
|
|
160884
|
+
|
|
160885
|
+
T compression_buffer[BITPACKING_WIDTH_GROUP_SIZE];
|
|
160886
|
+
bool compression_buffer_validity[BITPACKING_WIDTH_GROUP_SIZE];
|
|
160887
|
+
idx_t compression_buffer_idx;
|
|
160888
|
+
idx_t total_size;
|
|
160889
|
+
void *data_ptr;
|
|
160890
|
+
|
|
160891
|
+
public:
|
|
160892
|
+
template <class OP>
|
|
160893
|
+
void Flush() {
|
|
160894
|
+
bitpacking_width_t width = BitpackingPrimitives::MinimumBitWidth<T>(compression_buffer, compression_buffer_idx);
|
|
160895
|
+
OP::Operation(compression_buffer, compression_buffer_validity, width, compression_buffer_idx, data_ptr);
|
|
160896
|
+
total_size += (BITPACKING_WIDTH_GROUP_SIZE * width) / 8 + sizeof(bitpacking_width_t);
|
|
160897
|
+
compression_buffer_idx = 0;
|
|
160898
|
+
}
|
|
160899
|
+
|
|
160900
|
+
template <class OP = EmptyBitpackingWriter>
|
|
160901
|
+
void Update(T *data, ValidityMask &validity, idx_t idx) {
|
|
160902
|
+
|
|
160903
|
+
if (validity.RowIsValid(idx)) {
|
|
160904
|
+
compression_buffer_validity[compression_buffer_idx] = true;
|
|
160905
|
+
compression_buffer[compression_buffer_idx++] = data[idx];
|
|
160906
|
+
} else {
|
|
160907
|
+
// We write zero for easy bitwidth analysis of the compression buffer later
|
|
160908
|
+
compression_buffer_validity[compression_buffer_idx] = false;
|
|
160909
|
+
compression_buffer[compression_buffer_idx++] = 0;
|
|
160910
|
+
}
|
|
160911
|
+
|
|
160912
|
+
if (compression_buffer_idx == BITPACKING_WIDTH_GROUP_SIZE) {
|
|
160913
|
+
// Calculate bitpacking width;
|
|
160914
|
+
Flush<OP>();
|
|
160915
|
+
}
|
|
160916
|
+
}
|
|
160917
|
+
};
|
|
160918
|
+
|
|
160919
|
+
//===--------------------------------------------------------------------===//
|
|
160920
|
+
// Analyze
|
|
160921
|
+
//===--------------------------------------------------------------------===//
|
|
160922
|
+
template <class T>
|
|
160923
|
+
struct BitpackingAnalyzeState : public AnalyzeState {
|
|
160924
|
+
BitpackingState<T> state;
|
|
160925
|
+
};
|
|
160926
|
+
|
|
160927
|
+
template <class T>
|
|
160928
|
+
unique_ptr<AnalyzeState> BitpackingInitAnalyze(ColumnData &col_data, PhysicalType type) {
|
|
160929
|
+
return make_unique<BitpackingAnalyzeState<T>>();
|
|
160930
|
+
}
|
|
160931
|
+
|
|
160932
|
+
template <class T>
|
|
160933
|
+
bool BitpackingAnalyze(AnalyzeState &state, Vector &input, idx_t count) {
|
|
160934
|
+
auto &analyze_state = (BitpackingAnalyzeState<T> &)state;
|
|
160935
|
+
VectorData vdata;
|
|
160936
|
+
input.Orrify(count, vdata);
|
|
160937
|
+
|
|
160938
|
+
auto data = (T *)vdata.data;
|
|
160939
|
+
for (idx_t i = 0; i < count; i++) {
|
|
160940
|
+
auto idx = vdata.sel->get_index(i);
|
|
160941
|
+
analyze_state.state.template Update<EmptyBitpackingWriter>(data, vdata.validity, idx);
|
|
160942
|
+
}
|
|
160943
|
+
|
|
160944
|
+
return true;
|
|
160945
|
+
}
|
|
160946
|
+
|
|
160947
|
+
template <class T>
|
|
160948
|
+
idx_t BitpackingFinalAnalyze(AnalyzeState &state) {
|
|
160949
|
+
auto &bitpacking_state = (BitpackingAnalyzeState<T> &)state;
|
|
160950
|
+
bitpacking_state.state.template Flush<EmptyBitpackingWriter>();
|
|
160951
|
+
return bitpacking_state.state.total_size;
|
|
160952
|
+
}
|
|
160953
|
+
|
|
160954
|
+
//===--------------------------------------------------------------------===//
|
|
160955
|
+
// Compress
|
|
160956
|
+
//===--------------------------------------------------------------------===//
|
|
160957
|
+
template <class T>
|
|
160958
|
+
struct BitpackingCompressState : public CompressionState {
|
|
160959
|
+
public:
|
|
160960
|
+
explicit BitpackingCompressState(ColumnDataCheckpointer &checkpointer) : checkpointer(checkpointer) {
|
|
160961
|
+
auto &db = checkpointer.GetDatabase();
|
|
160962
|
+
auto &type = checkpointer.GetType();
|
|
160963
|
+
auto &config = DBConfig::GetConfig(db);
|
|
160964
|
+
function = config.GetCompressionFunction(CompressionType::COMPRESSION_BITPACKING, type.InternalType());
|
|
160965
|
+
CreateEmptySegment(checkpointer.GetRowGroup().start);
|
|
160966
|
+
|
|
160967
|
+
state.data_ptr = (void *)this;
|
|
160968
|
+
}
|
|
160969
|
+
|
|
160970
|
+
ColumnDataCheckpointer &checkpointer;
|
|
160971
|
+
CompressionFunction *function;
|
|
160972
|
+
unique_ptr<ColumnSegment> current_segment;
|
|
160973
|
+
unique_ptr<BufferHandle> handle;
|
|
160974
|
+
|
|
160975
|
+
// Ptr to next free spot in segment;
|
|
160976
|
+
data_ptr_t data_ptr;
|
|
160977
|
+
// Ptr to next free spot for storing bitwidths (growing downwards).
|
|
160978
|
+
data_ptr_t width_ptr;
|
|
160979
|
+
|
|
160980
|
+
BitpackingState<T> state;
|
|
160981
|
+
|
|
160982
|
+
public:
|
|
160983
|
+
struct BitpackingWriter {
|
|
160984
|
+
template <class VALUE_TYPE>
|
|
160985
|
+
static void Operation(VALUE_TYPE *values, bool *validity, bitpacking_width_t width, idx_t count,
|
|
160986
|
+
void *data_ptr) {
|
|
160987
|
+
auto state = (BitpackingCompressState<T> *)data_ptr;
|
|
160988
|
+
|
|
160989
|
+
if (state->RemainingSize() < (width * BITPACKING_WIDTH_GROUP_SIZE) / 8 + sizeof(bitpacking_width_t)) {
|
|
160990
|
+
// Segment is full
|
|
160991
|
+
auto row_start = state->current_segment->start + state->current_segment->count;
|
|
160992
|
+
state->FlushSegment();
|
|
160993
|
+
state->CreateEmptySegment(row_start);
|
|
160994
|
+
}
|
|
160995
|
+
|
|
160996
|
+
for (idx_t i = 0; i < count; i++) {
|
|
160997
|
+
if (validity[i]) {
|
|
160998
|
+
NumericStatistics::Update<T>(state->current_segment->stats, values[i]);
|
|
160999
|
+
}
|
|
161000
|
+
}
|
|
161001
|
+
|
|
161002
|
+
state->WriteValues(values, width, count);
|
|
161003
|
+
}
|
|
161004
|
+
};
|
|
161005
|
+
|
|
161006
|
+
// Space remaining between the width_ptr growing down and data ptr growing up
|
|
161007
|
+
idx_t RemainingSize() {
|
|
161008
|
+
return width_ptr - data_ptr;
|
|
161009
|
+
}
|
|
161010
|
+
|
|
161011
|
+
void CreateEmptySegment(idx_t row_start) {
|
|
161012
|
+
auto &db = checkpointer.GetDatabase();
|
|
161013
|
+
auto &type = checkpointer.GetType();
|
|
161014
|
+
auto compressed_segment = ColumnSegment::CreateTransientSegment(db, type, row_start);
|
|
161015
|
+
compressed_segment->function = function;
|
|
161016
|
+
current_segment = move(compressed_segment);
|
|
161017
|
+
auto &buffer_manager = BufferManager::GetBufferManager(db);
|
|
161018
|
+
handle = buffer_manager.Pin(current_segment->block);
|
|
161019
|
+
|
|
161020
|
+
data_ptr = handle->Ptr() + current_segment->GetBlockOffset() + BitpackingPrimitives::BITPACKING_HEADER_SIZE;
|
|
161021
|
+
width_ptr =
|
|
161022
|
+
handle->Ptr() + current_segment->GetBlockOffset() + Storage::BLOCK_SIZE - sizeof(bitpacking_width_t);
|
|
161023
|
+
}
|
|
161024
|
+
|
|
161025
|
+
void Append(VectorData &vdata, idx_t count) {
|
|
161026
|
+
// TODO Optimization: avoid use of compression buffer if we can compress straight to result vector
|
|
161027
|
+
auto data = (T *)vdata.data;
|
|
161028
|
+
|
|
161029
|
+
for (idx_t i = 0; i < count; i++) {
|
|
161030
|
+
auto idx = vdata.sel->get_index(i);
|
|
161031
|
+
state.template Update<BitpackingCompressState<T>::BitpackingWriter>(data, vdata.validity, idx);
|
|
161032
|
+
}
|
|
161033
|
+
}
|
|
161034
|
+
|
|
161035
|
+
void WriteValues(T *values, bitpacking_width_t width, idx_t count) {
|
|
161036
|
+
// TODO we can optimize this by stopping early if count < BITPACKING_WIDTH_GROUP_SIZE
|
|
161037
|
+
idx_t compress_loops = BITPACKING_WIDTH_GROUP_SIZE / BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE;
|
|
161038
|
+
for (idx_t i = 0; i < compress_loops; i++) {
|
|
161039
|
+
BitpackingPrimitives::PackBlock<T>(
|
|
161040
|
+
data_ptr, &values[i * BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE], width);
|
|
161041
|
+
data_ptr += (BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE * width) / 8;
|
|
161042
|
+
}
|
|
161043
|
+
|
|
161044
|
+
Store<bitpacking_width_t>(width, width_ptr);
|
|
161045
|
+
width_ptr -= sizeof(bitpacking_width_t);
|
|
161046
|
+
|
|
161047
|
+
current_segment->count += count;
|
|
161048
|
+
}
|
|
161049
|
+
|
|
161050
|
+
void FlushSegment() {
|
|
161051
|
+
auto &state = checkpointer.GetCheckpointState();
|
|
161052
|
+
|
|
161053
|
+
// Compact the segment by moving the widths next to the data.
|
|
161054
|
+
idx_t minimal_widths_offset = AlignValue(data_ptr - handle->node->buffer);
|
|
161055
|
+
idx_t widths_size = handle->node->buffer + Storage::BLOCK_SIZE - width_ptr - 1;
|
|
161056
|
+
idx_t total_segment_size = minimal_widths_offset + widths_size;
|
|
161057
|
+
memmove(handle->node->buffer + minimal_widths_offset, width_ptr + 1, widths_size);
|
|
161058
|
+
|
|
161059
|
+
// Store the offset of the first width (which is at the highest address).
|
|
161060
|
+
Store<idx_t>(minimal_widths_offset + widths_size - 1, handle->node->buffer);
|
|
161061
|
+
handle.reset();
|
|
161062
|
+
|
|
161063
|
+
state.FlushSegment(move(current_segment), total_segment_size);
|
|
161064
|
+
}
|
|
161065
|
+
|
|
161066
|
+
void Finalize() {
|
|
161067
|
+
state.template Flush<BitpackingCompressState<T>::BitpackingWriter>();
|
|
161068
|
+
FlushSegment();
|
|
161069
|
+
current_segment.reset();
|
|
161070
|
+
}
|
|
161071
|
+
};
|
|
161072
|
+
|
|
161073
|
+
template <class T>
|
|
161074
|
+
unique_ptr<CompressionState> BitpackingInitCompression(ColumnDataCheckpointer &checkpointer,
|
|
161075
|
+
unique_ptr<AnalyzeState> state) {
|
|
161076
|
+
return make_unique<BitpackingCompressState<T>>(checkpointer);
|
|
161077
|
+
}
|
|
161078
|
+
|
|
161079
|
+
template <class T>
|
|
161080
|
+
void BitpackingCompress(CompressionState &state_p, Vector &scan_vector, idx_t count) {
|
|
161081
|
+
auto &state = (BitpackingCompressState<T> &)state_p;
|
|
161082
|
+
VectorData vdata;
|
|
161083
|
+
scan_vector.Orrify(count, vdata);
|
|
161084
|
+
state.Append(vdata, count);
|
|
161085
|
+
}
|
|
161086
|
+
|
|
161087
|
+
template <class T>
|
|
161088
|
+
void BitpackingFinalizeCompress(CompressionState &state_p) {
|
|
161089
|
+
auto &state = (BitpackingCompressState<T> &)state_p;
|
|
161090
|
+
state.Finalize();
|
|
161091
|
+
}
|
|
161092
|
+
|
|
161093
|
+
//===--------------------------------------------------------------------===//
|
|
161094
|
+
// Scan
|
|
161095
|
+
//===--------------------------------------------------------------------===//
|
|
161096
|
+
template <class T>
|
|
161097
|
+
struct BitpackingScanState : public SegmentScanState {
|
|
161098
|
+
public:
|
|
161099
|
+
explicit BitpackingScanState(ColumnSegment &segment) {
|
|
161100
|
+
auto &buffer_manager = BufferManager::GetBufferManager(segment.db);
|
|
161101
|
+
handle = buffer_manager.Pin(segment.block);
|
|
161102
|
+
|
|
161103
|
+
current_width_group_ptr =
|
|
161104
|
+
handle->node->buffer + segment.GetBlockOffset() + BitpackingPrimitives::BITPACKING_HEADER_SIZE;
|
|
161105
|
+
|
|
161106
|
+
// load offset to bitpacking widths pointer
|
|
161107
|
+
auto bitpacking_widths_offset = Load<idx_t>(handle->node->buffer + segment.GetBlockOffset());
|
|
161108
|
+
bitpacking_width_ptr = handle->node->buffer + segment.GetBlockOffset() + bitpacking_widths_offset;
|
|
161109
|
+
|
|
161110
|
+
// load the bitwidth of the first vector
|
|
161111
|
+
LoadCurrentBitWidth();
|
|
161112
|
+
}
|
|
161113
|
+
|
|
161114
|
+
unique_ptr<BufferHandle> handle;
|
|
161115
|
+
|
|
161116
|
+
void (*decompress_function)(data_ptr_t, data_ptr_t, bitpacking_width_t, bool skip_sign_extension);
|
|
161117
|
+
T decompression_buffer[BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE];
|
|
161118
|
+
|
|
161119
|
+
idx_t position_in_group = 0;
|
|
161120
|
+
data_ptr_t current_width_group_ptr;
|
|
161121
|
+
data_ptr_t bitpacking_width_ptr;
|
|
161122
|
+
bitpacking_width_t current_width;
|
|
161123
|
+
|
|
161124
|
+
public:
|
|
161125
|
+
void LoadCurrentBitWidth() {
|
|
161126
|
+
D_ASSERT(bitpacking_width_ptr > handle->node->buffer &&
|
|
161127
|
+
bitpacking_width_ptr < handle->node->buffer + Storage::BLOCK_SIZE);
|
|
161128
|
+
current_width = Load<bitpacking_width_t>(bitpacking_width_ptr);
|
|
161129
|
+
LoadDecompressFunction();
|
|
161130
|
+
}
|
|
161131
|
+
|
|
161132
|
+
void Skip(ColumnSegment &segment, idx_t skip_count) {
|
|
161133
|
+
while (skip_count > 0) {
|
|
161134
|
+
if (position_in_group + skip_count < BITPACKING_WIDTH_GROUP_SIZE) {
|
|
161135
|
+
// We're not leaving this bitpacking group, we can perform all skips.
|
|
161136
|
+
position_in_group += skip_count;
|
|
161137
|
+
break;
|
|
161138
|
+
} else {
|
|
161139
|
+
// The skip crosses the current bitpacking group, we skip the remainder of this group.
|
|
161140
|
+
auto skipping = BITPACKING_WIDTH_GROUP_SIZE - position_in_group;
|
|
161141
|
+
position_in_group = 0;
|
|
161142
|
+
current_width_group_ptr += (current_width * BITPACKING_WIDTH_GROUP_SIZE) / 8;
|
|
161143
|
+
|
|
161144
|
+
// Update width pointer and load new width
|
|
161145
|
+
bitpacking_width_ptr -= sizeof(bitpacking_width_t);
|
|
161146
|
+
LoadCurrentBitWidth();
|
|
161147
|
+
|
|
161148
|
+
skip_count -= skipping;
|
|
161149
|
+
}
|
|
161150
|
+
}
|
|
161151
|
+
}
|
|
161152
|
+
|
|
161153
|
+
void LoadDecompressFunction() {
|
|
161154
|
+
decompress_function = &BitpackingPrimitives::UnPackBlock<T>;
|
|
161155
|
+
}
|
|
161156
|
+
};
|
|
161157
|
+
|
|
161158
|
+
template <class T>
|
|
161159
|
+
unique_ptr<SegmentScanState> BitpackingInitScan(ColumnSegment &segment) {
|
|
161160
|
+
auto result = make_unique<BitpackingScanState<T>>(segment);
|
|
161161
|
+
return move(result);
|
|
161162
|
+
}
|
|
161163
|
+
|
|
161164
|
+
//===--------------------------------------------------------------------===//
|
|
161165
|
+
// Scan base data
|
|
161166
|
+
//===--------------------------------------------------------------------===//
|
|
161167
|
+
template <class T>
|
|
161168
|
+
void BitpackingScanPartial(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result,
|
|
161169
|
+
idx_t result_offset) {
|
|
161170
|
+
auto &scan_state = (BitpackingScanState<T> &)*state.scan_state;
|
|
161171
|
+
|
|
161172
|
+
T *result_data = FlatVector::GetData<T>(result);
|
|
161173
|
+
result.SetVectorType(VectorType::FLAT_VECTOR);
|
|
161174
|
+
|
|
161175
|
+
// Fast path for when no compression was used, we can do a single memcopy
|
|
161176
|
+
if (STANDARD_VECTOR_SIZE == BITPACKING_WIDTH_GROUP_SIZE) {
|
|
161177
|
+
if (scan_state.current_width == sizeof(T) * 8 && scan_count <= BITPACKING_WIDTH_GROUP_SIZE &&
|
|
161178
|
+
scan_state.position_in_group == 0) {
|
|
161179
|
+
|
|
161180
|
+
memcpy(result_data + result_offset, scan_state.current_width_group_ptr, scan_count * sizeof(T));
|
|
161181
|
+
scan_state.current_width_group_ptr += scan_count * sizeof(T);
|
|
161182
|
+
scan_state.bitpacking_width_ptr -= sizeof(bitpacking_width_t);
|
|
161183
|
+
scan_state.LoadCurrentBitWidth();
|
|
161184
|
+
return;
|
|
161185
|
+
}
|
|
161186
|
+
}
|
|
161187
|
+
|
|
161188
|
+
// Determine if we can skip sign extension during compression
|
|
161189
|
+
auto &nstats = (NumericStatistics &)*segment.stats.statistics;
|
|
161190
|
+
bool skip_sign_extend = std::is_signed<T>::value && nstats.min >= 0;
|
|
161191
|
+
|
|
161192
|
+
idx_t scanned = 0;
|
|
161193
|
+
|
|
161194
|
+
while (scanned < scan_count) {
|
|
161195
|
+
// Exhausted this width group, move pointers to next group and load bitwidth for next group.
|
|
161196
|
+
if (scan_state.position_in_group >= BITPACKING_WIDTH_GROUP_SIZE) {
|
|
161197
|
+
scan_state.position_in_group = 0;
|
|
161198
|
+
scan_state.bitpacking_width_ptr -= sizeof(bitpacking_width_t);
|
|
161199
|
+
scan_state.current_width_group_ptr += (scan_state.current_width * BITPACKING_WIDTH_GROUP_SIZE) / 8;
|
|
161200
|
+
scan_state.LoadCurrentBitWidth();
|
|
161201
|
+
}
|
|
161202
|
+
|
|
161203
|
+
idx_t offset_in_compression_group =
|
|
161204
|
+
scan_state.position_in_group % BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE;
|
|
161205
|
+
|
|
161206
|
+
idx_t to_scan = MinValue<idx_t>(scan_count - scanned, BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE -
|
|
161207
|
+
offset_in_compression_group);
|
|
161208
|
+
|
|
161209
|
+
// Calculate start of compression algorithm group
|
|
161210
|
+
data_ptr_t current_position_ptr =
|
|
161211
|
+
scan_state.current_width_group_ptr + scan_state.position_in_group * scan_state.current_width / 8;
|
|
161212
|
+
data_ptr_t decompression_group_start_pointer =
|
|
161213
|
+
current_position_ptr - offset_in_compression_group * scan_state.current_width / 8;
|
|
161214
|
+
|
|
161215
|
+
T *current_result_ptr = result_data + result_offset + scanned;
|
|
161216
|
+
|
|
161217
|
+
if (to_scan == BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE && offset_in_compression_group == 0) {
|
|
161218
|
+
// Decompress directly into result vector
|
|
161219
|
+
scan_state.decompress_function((data_ptr_t)current_result_ptr, decompression_group_start_pointer,
|
|
161220
|
+
scan_state.current_width, skip_sign_extend);
|
|
161221
|
+
} else {
|
|
161222
|
+
// Decompress compression algorithm to buffer
|
|
161223
|
+
scan_state.decompress_function((data_ptr_t)scan_state.decompression_buffer,
|
|
161224
|
+
decompression_group_start_pointer, scan_state.current_width,
|
|
161225
|
+
skip_sign_extend);
|
|
161226
|
+
|
|
161227
|
+
memcpy(current_result_ptr, scan_state.decompression_buffer + offset_in_compression_group,
|
|
161228
|
+
to_scan * sizeof(T));
|
|
161229
|
+
}
|
|
161230
|
+
|
|
161231
|
+
scanned += to_scan;
|
|
161232
|
+
scan_state.position_in_group += to_scan;
|
|
161233
|
+
}
|
|
161234
|
+
}
|
|
161235
|
+
|
|
161236
|
+
template <class T>
|
|
161237
|
+
void BitpackingScan(ColumnSegment &segment, ColumnScanState &state, idx_t scan_count, Vector &result) {
|
|
161238
|
+
BitpackingScanPartial<T>(segment, state, scan_count, result, 0);
|
|
161239
|
+
}
|
|
161240
|
+
|
|
161241
|
+
//===--------------------------------------------------------------------===//
|
|
161242
|
+
// Fetch
|
|
161243
|
+
//===--------------------------------------------------------------------===//
|
|
161244
|
+
template <class T>
|
|
161245
|
+
void BitpackingFetchRow(ColumnSegment &segment, ColumnFetchState &state, row_t row_id, Vector &result,
|
|
161246
|
+
idx_t result_idx) {
|
|
161247
|
+
BitpackingScanState<T> scan_state(segment);
|
|
161248
|
+
scan_state.Skip(segment, row_id);
|
|
161249
|
+
auto result_data = FlatVector::GetData<T>(result);
|
|
161250
|
+
T *current_result_ptr = result_data + result_idx;
|
|
161251
|
+
|
|
161252
|
+
// TODO clean up, is reused in partialscan
|
|
161253
|
+
idx_t offset_in_compression_group =
|
|
161254
|
+
scan_state.position_in_group % BitpackingPrimitives::BITPACKING_ALGORITHM_GROUP_SIZE;
|
|
161255
|
+
|
|
161256
|
+
data_ptr_t decompression_group_start_pointer =
|
|
161257
|
+
scan_state.current_width_group_ptr +
|
|
161258
|
+
(scan_state.position_in_group - offset_in_compression_group) * scan_state.current_width / 8;
|
|
161259
|
+
|
|
161260
|
+
auto &nstats = (NumericStatistics &)*segment.stats.statistics;
|
|
161261
|
+
bool skip_sign_extend = std::is_signed<T>::value && nstats.min >= 0;
|
|
161262
|
+
|
|
161263
|
+
scan_state.decompress_function((data_ptr_t)scan_state.decompression_buffer, decompression_group_start_pointer,
|
|
161264
|
+
scan_state.current_width, skip_sign_extend);
|
|
161265
|
+
|
|
161266
|
+
*current_result_ptr = *(T *)(scan_state.decompression_buffer + offset_in_compression_group);
|
|
161267
|
+
}
|
|
161268
|
+
template <class T>
|
|
161269
|
+
void BitpackingSkip(ColumnSegment &segment, ColumnScanState &state, idx_t skip_count) {
|
|
161270
|
+
auto &scan_state = (BitpackingScanState<T> &)*state.scan_state;
|
|
161271
|
+
scan_state.Skip(segment, skip_count);
|
|
161272
|
+
}
|
|
161273
|
+
|
|
161274
|
+
//===--------------------------------------------------------------------===//
|
|
161275
|
+
// Get Function
|
|
161276
|
+
//===--------------------------------------------------------------------===//
|
|
161277
|
+
template <class T>
|
|
161278
|
+
CompressionFunction GetBitpackingFunction(PhysicalType data_type) {
|
|
161279
|
+
return CompressionFunction(CompressionType::COMPRESSION_BITPACKING, data_type, BitpackingInitAnalyze<T>,
|
|
161280
|
+
BitpackingAnalyze<T>, BitpackingFinalAnalyze<T>, BitpackingInitCompression<T>,
|
|
161281
|
+
BitpackingCompress<T>, BitpackingFinalizeCompress<T>, BitpackingInitScan<T>,
|
|
161282
|
+
BitpackingScan<T>, BitpackingScanPartial<T>, BitpackingFetchRow<T>, BitpackingSkip<T>);
|
|
161283
|
+
}
|
|
161284
|
+
|
|
161285
|
+
CompressionFunction BitpackingFun::GetFunction(PhysicalType type) {
|
|
161286
|
+
switch (type) {
|
|
161287
|
+
case PhysicalType::INT8:
|
|
161288
|
+
return GetBitpackingFunction<int8_t>(type);
|
|
161289
|
+
case PhysicalType::INT16:
|
|
161290
|
+
return GetBitpackingFunction<int16_t>(type);
|
|
161291
|
+
case PhysicalType::INT32:
|
|
161292
|
+
return GetBitpackingFunction<int32_t>(type);
|
|
161293
|
+
case PhysicalType::INT64:
|
|
161294
|
+
return GetBitpackingFunction<int64_t>(type);
|
|
161295
|
+
case PhysicalType::BOOL:
|
|
161296
|
+
case PhysicalType::UINT8:
|
|
161297
|
+
return GetBitpackingFunction<uint8_t>(type);
|
|
161298
|
+
case PhysicalType::UINT16:
|
|
161299
|
+
return GetBitpackingFunction<uint16_t>(type);
|
|
161300
|
+
case PhysicalType::UINT32:
|
|
161301
|
+
return GetBitpackingFunction<uint32_t>(type);
|
|
161302
|
+
case PhysicalType::UINT64:
|
|
161303
|
+
return GetBitpackingFunction<uint64_t>(type);
|
|
161304
|
+
default:
|
|
161305
|
+
throw InternalException("Unsupported type for Bitpacking");
|
|
161306
|
+
}
|
|
161307
|
+
}
|
|
161308
|
+
|
|
161309
|
+
bool BitpackingFun::TypeIsSupported(PhysicalType type) {
|
|
161310
|
+
switch (type) {
|
|
161311
|
+
case PhysicalType::BOOL:
|
|
161312
|
+
case PhysicalType::INT8:
|
|
161313
|
+
case PhysicalType::INT16:
|
|
161314
|
+
case PhysicalType::INT32:
|
|
161315
|
+
case PhysicalType::INT64:
|
|
161316
|
+
case PhysicalType::UINT8:
|
|
161317
|
+
case PhysicalType::UINT16:
|
|
161318
|
+
case PhysicalType::UINT32:
|
|
161319
|
+
case PhysicalType::UINT64:
|
|
161320
|
+
return true;
|
|
161321
|
+
default:
|
|
161322
|
+
return false;
|
|
161323
|
+
}
|
|
161324
|
+
}
|
|
161325
|
+
|
|
161326
|
+
} // namespace duckdb
|
|
161327
|
+
//===----------------------------------------------------------------------===//
|
|
161328
|
+
// DuckDB
|
|
161329
|
+
//
|
|
161330
|
+
// duckdb/storage/segment/uncompressed.hpp
|
|
161331
|
+
//
|
|
161332
|
+
//
|
|
161333
|
+
//===----------------------------------------------------------------------===//
|
|
161334
|
+
|
|
161335
|
+
|
|
161336
|
+
|
|
161337
|
+
|
|
161338
|
+
|
|
161339
|
+
namespace duckdb {
|
|
161340
|
+
class DatabaseInstance;
|
|
161341
|
+
|
|
161342
|
+
struct UncompressedFunctions {
|
|
161343
|
+
static unique_ptr<CompressionState> InitCompression(ColumnDataCheckpointer &checkpointer,
|
|
161344
|
+
unique_ptr<AnalyzeState> state);
|
|
161345
|
+
static void Compress(CompressionState &state_p, Vector &data, idx_t count);
|
|
161346
|
+
static void FinalizeCompress(CompressionState &state_p);
|
|
161347
|
+
static void EmptySkip(ColumnSegment &segment, ColumnScanState &state, idx_t skip_count) {
|
|
161348
|
+
}
|
|
161349
|
+
};
|
|
161350
|
+
|
|
161351
|
+
struct FixedSizeUncompressed {
|
|
161352
|
+
static CompressionFunction GetFunction(PhysicalType data_type);
|
|
161353
|
+
};
|
|
161354
|
+
|
|
161355
|
+
struct ValidityUncompressed {
|
|
161356
|
+
public:
|
|
161357
|
+
static CompressionFunction GetFunction(PhysicalType data_type);
|
|
161358
|
+
|
|
161359
|
+
public:
|
|
161360
|
+
static const validity_t LOWER_MASKS[65];
|
|
161361
|
+
static const validity_t UPPER_MASKS[65];
|
|
161362
|
+
};
|
|
161363
|
+
|
|
161364
|
+
struct StringUncompressed {
|
|
161365
|
+
public:
|
|
161366
|
+
static CompressionFunction GetFunction(PhysicalType data_type);
|
|
161367
|
+
|
|
161368
|
+
public:
|
|
161369
|
+
//! The max string size that is allowed within a block. Strings bigger than this will be labeled as a BIG STRING and
|
|
161370
|
+
//! offloaded to the overflow blocks.
|
|
161371
|
+
static constexpr uint16_t STRING_BLOCK_LIMIT = 4096;
|
|
161372
|
+
};
|
|
161373
|
+
|
|
161374
|
+
} // namespace duckdb
|
|
161375
|
+
|
|
161376
|
+
|
|
161377
|
+
|
|
161378
|
+
|
|
161379
|
+
|
|
161380
|
+
|
|
161381
|
+
|
|
161382
|
+
|
|
161383
|
+
|
|
161384
|
+
|
|
161385
|
+
|
|
161386
|
+
|
|
159581
161387
|
namespace duckdb {
|
|
159582
161388
|
|
|
159583
161389
|
//===--------------------------------------------------------------------===//
|
|
@@ -207501,6 +209307,1299 @@ void sdfree(void *ptr) { free(ptr); }
|
|
|
207501
209307
|
// LICENSE_CHANGE_END
|
|
207502
209308
|
|
|
207503
209309
|
|
|
209310
|
+
// LICENSE_CHANGE_BEGIN
|
|
209311
|
+
// The following code up to LICENSE_CHANGE_END is subject to THIRD PARTY LICENSE #10
|
|
209312
|
+
// See the end of this file for a list
|
|
209313
|
+
|
|
209314
|
+
|
|
209315
|
+
|
|
209316
|
+
#include <cstdint>
|
|
209317
|
+
#include <type_traits>
|
|
209318
|
+
|
|
209319
|
+
namespace duckdb_fastpforlib {
|
|
209320
|
+
namespace internal {
|
|
209321
|
+
|
|
209322
|
+
// Used for uint8_t, uint16_t and uint32_t
|
|
209323
|
+
template <uint8_t DELTA, uint8_t SHR, class TYPE, uint8_t TYPE_SIZE = sizeof(TYPE) * 8>
|
|
209324
|
+
typename std::enable_if<(DELTA + SHR) < TYPE_SIZE>::type unpack_single_out(const TYPE *__restrict in,
|
|
209325
|
+
TYPE *__restrict out) {
|
|
209326
|
+
*out = ((*in) >> SHR) % (1 << DELTA);
|
|
209327
|
+
}
|
|
209328
|
+
|
|
209329
|
+
// Used for uint8_t, uint16_t and uint32_t
|
|
209330
|
+
template <uint8_t DELTA, uint8_t SHR, class TYPE, uint8_t TYPE_SIZE = sizeof(TYPE) * 8>
|
|
209331
|
+
typename std::enable_if<(DELTA + SHR) >= TYPE_SIZE>::type unpack_single_out(const TYPE *__restrict &in,
|
|
209332
|
+
TYPE *__restrict out) {
|
|
209333
|
+
*out = (*in) >> SHR;
|
|
209334
|
+
++in;
|
|
209335
|
+
|
|
209336
|
+
static const TYPE NEXT_SHR = SHR + DELTA - TYPE_SIZE;
|
|
209337
|
+
*out |= ((*in) % (1U << NEXT_SHR)) << (TYPE_SIZE - SHR);
|
|
209338
|
+
}
|
|
209339
|
+
|
|
209340
|
+
template <uint8_t DELTA, uint8_t SHR>
|
|
209341
|
+
typename std::enable_if<(DELTA + SHR) < 32>::type unpack_single_out(const uint32_t *__restrict in,
|
|
209342
|
+
uint64_t *__restrict out) {
|
|
209343
|
+
*out = ((static_cast<uint64_t>(*in)) >> SHR) % (1ULL << DELTA);
|
|
209344
|
+
}
|
|
209345
|
+
|
|
209346
|
+
template <uint8_t DELTA, uint8_t SHR>
|
|
209347
|
+
typename std::enable_if<(DELTA + SHR) >= 32 && (DELTA + SHR) < 64>::type
|
|
209348
|
+
unpack_single_out(const uint32_t *__restrict &in, uint64_t *__restrict out) {
|
|
209349
|
+
*out = static_cast<uint64_t>(*in) >> SHR;
|
|
209350
|
+
++in;
|
|
209351
|
+
if (DELTA + SHR > 32) {
|
|
209352
|
+
static const uint8_t NEXT_SHR = SHR + DELTA - 32;
|
|
209353
|
+
*out |= static_cast<uint64_t>((*in) % (1U << NEXT_SHR)) << (32 - SHR);
|
|
209354
|
+
}
|
|
209355
|
+
}
|
|
209356
|
+
|
|
209357
|
+
template <uint8_t DELTA, uint8_t SHR>
|
|
209358
|
+
typename std::enable_if<(DELTA + SHR) >= 64>::type unpack_single_out(const uint32_t *__restrict &in,
|
|
209359
|
+
uint64_t *__restrict out) {
|
|
209360
|
+
*out = static_cast<uint64_t>(*in) >> SHR;
|
|
209361
|
+
++in;
|
|
209362
|
+
|
|
209363
|
+
*out |= static_cast<uint64_t>(*in) << (32 - SHR);
|
|
209364
|
+
++in;
|
|
209365
|
+
|
|
209366
|
+
if (DELTA + SHR > 64) {
|
|
209367
|
+
static const uint8_t NEXT_SHR = DELTA + SHR - 64;
|
|
209368
|
+
*out |= static_cast<uint64_t>((*in) % (1U << NEXT_SHR)) << (64 - SHR);
|
|
209369
|
+
}
|
|
209370
|
+
}
|
|
209371
|
+
|
|
209372
|
+
// Used for uint8_t, uint16_t and uint32_t
|
|
209373
|
+
template <class TYPE, uint16_t DELTA, uint16_t SHL, TYPE MASK, uint8_t TYPE_SIZE = sizeof(TYPE) * 8>
|
|
209374
|
+
typename std::enable_if < DELTA + SHL<TYPE_SIZE>::type pack_single_in(const TYPE in, TYPE *__restrict out) {
|
|
209375
|
+
if (SHL == 0) {
|
|
209376
|
+
*out = in & MASK;
|
|
209377
|
+
} else {
|
|
209378
|
+
*out |= (in & MASK) << SHL;
|
|
209379
|
+
}
|
|
209380
|
+
}
|
|
209381
|
+
|
|
209382
|
+
// Used for uint8_t, uint16_t and uint32_t
|
|
209383
|
+
template <class TYPE, uint16_t DELTA, uint16_t SHL, TYPE MASK, uint8_t TYPE_SIZE = sizeof(TYPE) * 8>
|
|
209384
|
+
typename std::enable_if<DELTA + SHL >= TYPE_SIZE>::type pack_single_in(const TYPE in, TYPE *__restrict &out) {
|
|
209385
|
+
*out |= in << SHL;
|
|
209386
|
+
++out;
|
|
209387
|
+
|
|
209388
|
+
if (DELTA + SHL > TYPE_SIZE) {
|
|
209389
|
+
*out = (in & MASK) >> (TYPE_SIZE - SHL);
|
|
209390
|
+
}
|
|
209391
|
+
}
|
|
209392
|
+
|
|
209393
|
+
template <uint16_t DELTA, uint16_t SHL, uint64_t MASK>
|
|
209394
|
+
typename std::enable_if < DELTA + SHL<32>::type pack_single_in64(const uint64_t in, uint32_t *__restrict out) {
|
|
209395
|
+
if (SHL == 0) {
|
|
209396
|
+
*out = static_cast<uint32_t>(in & MASK);
|
|
209397
|
+
} else {
|
|
209398
|
+
*out |= (in & MASK) << SHL;
|
|
209399
|
+
}
|
|
209400
|
+
}
|
|
209401
|
+
template <uint16_t DELTA, uint16_t SHL, uint64_t MASK>
|
|
209402
|
+
typename std::enable_if < DELTA + SHL >= 32 &&
|
|
209403
|
+
DELTA + SHL<64>::type pack_single_in64(const uint64_t in, uint32_t *__restrict &out) {
|
|
209404
|
+
if (SHL == 0) {
|
|
209405
|
+
*out = static_cast<uint32_t>(in & MASK);
|
|
209406
|
+
} else {
|
|
209407
|
+
*out |= (in & MASK) << SHL;
|
|
209408
|
+
}
|
|
209409
|
+
|
|
209410
|
+
++out;
|
|
209411
|
+
|
|
209412
|
+
if (DELTA + SHL > 32) {
|
|
209413
|
+
*out = static_cast<uint32_t>((in & MASK) >> (32 - SHL));
|
|
209414
|
+
}
|
|
209415
|
+
}
|
|
209416
|
+
template <uint16_t DELTA, uint16_t SHL, uint64_t MASK>
|
|
209417
|
+
typename std::enable_if<DELTA + SHL >= 64>::type pack_single_in64(const uint64_t in, uint32_t *__restrict &out) {
|
|
209418
|
+
*out |= in << SHL;
|
|
209419
|
+
++out;
|
|
209420
|
+
|
|
209421
|
+
*out = static_cast<uint32_t>((in & MASK) >> (32 - SHL));
|
|
209422
|
+
++out;
|
|
209423
|
+
|
|
209424
|
+
if (DELTA + SHL > 64) {
|
|
209425
|
+
*out = (in & MASK) >> (64 - SHL);
|
|
209426
|
+
}
|
|
209427
|
+
}
|
|
209428
|
+
template <uint16_t DELTA, uint16_t OINDEX = 0>
|
|
209429
|
+
struct Unroller8 {
|
|
209430
|
+
static void Unpack(const uint8_t *__restrict &in, uint8_t *__restrict out) {
|
|
209431
|
+
unpack_single_out<DELTA, (DELTA * OINDEX) % 8>(in, out + OINDEX);
|
|
209432
|
+
|
|
209433
|
+
Unroller8<DELTA, OINDEX + 1>::Unpack(in, out);
|
|
209434
|
+
}
|
|
209435
|
+
|
|
209436
|
+
static void Pack(const uint8_t *__restrict in, uint8_t *__restrict out) {
|
|
209437
|
+
pack_single_in<uint8_t, DELTA, (DELTA * OINDEX) % 8, (1U << DELTA) - 1>(in[OINDEX], out);
|
|
209438
|
+
|
|
209439
|
+
Unroller8<DELTA, OINDEX + 1>::Pack(in, out);
|
|
209440
|
+
}
|
|
209441
|
+
|
|
209442
|
+
};\
|
|
209443
|
+
template <uint16_t DELTA>
|
|
209444
|
+
struct Unroller8<DELTA, 7> {
|
|
209445
|
+
enum { SHIFT = (DELTA * 7) % 8 };
|
|
209446
|
+
|
|
209447
|
+
static void Unpack(const uint8_t *__restrict in, uint8_t *__restrict out) {
|
|
209448
|
+
out[7] = (*in) >> SHIFT;
|
|
209449
|
+
}
|
|
209450
|
+
|
|
209451
|
+
static void Pack(const uint8_t *__restrict in, uint8_t *__restrict out) {
|
|
209452
|
+
*out |= (in[7] << SHIFT);
|
|
209453
|
+
}
|
|
209454
|
+
};
|
|
209455
|
+
|
|
209456
|
+
template <uint16_t DELTA, uint16_t OINDEX = 0>
|
|
209457
|
+
struct Unroller16 {
|
|
209458
|
+
static void Unpack(const uint16_t *__restrict &in, uint16_t *__restrict out) {
|
|
209459
|
+
unpack_single_out<DELTA, (DELTA * OINDEX) % 16>(in, out + OINDEX);
|
|
209460
|
+
|
|
209461
|
+
Unroller16<DELTA, OINDEX + 1>::Unpack(in, out);
|
|
209462
|
+
}
|
|
209463
|
+
|
|
209464
|
+
static void Pack(const uint16_t *__restrict in, uint16_t *__restrict out) {
|
|
209465
|
+
pack_single_in<uint16_t, DELTA, (DELTA * OINDEX) % 16, (1U << DELTA) - 1>(in[OINDEX], out);
|
|
209466
|
+
|
|
209467
|
+
Unroller16<DELTA, OINDEX + 1>::Pack(in, out);
|
|
209468
|
+
}
|
|
209469
|
+
|
|
209470
|
+
};
|
|
209471
|
+
|
|
209472
|
+
template <uint16_t DELTA>
|
|
209473
|
+
struct Unroller16<DELTA, 15> {
|
|
209474
|
+
enum { SHIFT = (DELTA * 15) % 16 };
|
|
209475
|
+
|
|
209476
|
+
static void Unpack(const uint16_t *__restrict in, uint16_t *__restrict out) {
|
|
209477
|
+
out[15] = (*in) >> SHIFT;
|
|
209478
|
+
}
|
|
209479
|
+
|
|
209480
|
+
static void Pack(const uint16_t *__restrict in, uint16_t *__restrict out) {
|
|
209481
|
+
*out |= (in[15] << SHIFT);
|
|
209482
|
+
}
|
|
209483
|
+
};
|
|
209484
|
+
|
|
209485
|
+
template <uint16_t DELTA, uint16_t OINDEX = 0>
|
|
209486
|
+
struct Unroller {
|
|
209487
|
+
static void Unpack(const uint32_t *__restrict &in, uint32_t *__restrict out) {
|
|
209488
|
+
unpack_single_out<DELTA, (DELTA * OINDEX) % 32>(in, out + OINDEX);
|
|
209489
|
+
|
|
209490
|
+
Unroller<DELTA, OINDEX + 1>::Unpack(in, out);
|
|
209491
|
+
}
|
|
209492
|
+
|
|
209493
|
+
static void Unpack(const uint32_t *__restrict &in, uint64_t *__restrict out) {
|
|
209494
|
+
unpack_single_out<DELTA, (DELTA * OINDEX) % 32>(in, out + OINDEX);
|
|
209495
|
+
|
|
209496
|
+
Unroller<DELTA, OINDEX + 1>::Unpack(in, out);
|
|
209497
|
+
}
|
|
209498
|
+
|
|
209499
|
+
static void Pack(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
209500
|
+
pack_single_in<uint32_t, DELTA, (DELTA * OINDEX) % 32, (1U << DELTA) - 1>(in[OINDEX], out);
|
|
209501
|
+
|
|
209502
|
+
Unroller<DELTA, OINDEX + 1>::Pack(in, out);
|
|
209503
|
+
}
|
|
209504
|
+
|
|
209505
|
+
static void Pack(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
209506
|
+
pack_single_in64<DELTA, (DELTA * OINDEX) % 32, (1ULL << DELTA) - 1>(in[OINDEX], out);
|
|
209507
|
+
|
|
209508
|
+
Unroller<DELTA, OINDEX + 1>::Pack(in, out);
|
|
209509
|
+
}
|
|
209510
|
+
};
|
|
209511
|
+
|
|
209512
|
+
template <uint16_t DELTA>
|
|
209513
|
+
struct Unroller<DELTA, 31> {
|
|
209514
|
+
enum { SHIFT = (DELTA * 31) % 32 };
|
|
209515
|
+
|
|
209516
|
+
static void Unpack(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
209517
|
+
out[31] = (*in) >> SHIFT;
|
|
209518
|
+
}
|
|
209519
|
+
|
|
209520
|
+
static void Unpack(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209521
|
+
out[31] = (*in) >> SHIFT;
|
|
209522
|
+
if (DELTA > 32) {
|
|
209523
|
+
++in;
|
|
209524
|
+
out[31] |= static_cast<uint64_t>(*in) << (32 - SHIFT);
|
|
209525
|
+
}
|
|
209526
|
+
}
|
|
209527
|
+
|
|
209528
|
+
static void Pack(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
209529
|
+
*out |= (in[31] << SHIFT);
|
|
209530
|
+
}
|
|
209531
|
+
|
|
209532
|
+
static void Pack(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
209533
|
+
*out |= (in[31] << SHIFT);
|
|
209534
|
+
if (DELTA > 32) {
|
|
209535
|
+
++out;
|
|
209536
|
+
*out = static_cast<uint32_t>(in[31] >> (32 - SHIFT));
|
|
209537
|
+
}
|
|
209538
|
+
}
|
|
209539
|
+
};
|
|
209540
|
+
|
|
209541
|
+
// Special cases
|
|
209542
|
+
void __fastunpack0(const uint8_t *__restrict, uint8_t *__restrict out) {
|
|
209543
|
+
for (uint8_t i = 0; i < 8; ++i)
|
|
209544
|
+
*(out++) = 0;
|
|
209545
|
+
}
|
|
209546
|
+
|
|
209547
|
+
void __fastunpack0(const uint16_t *__restrict, uint16_t *__restrict out) {
|
|
209548
|
+
for (uint16_t i = 0; i < 16; ++i)
|
|
209549
|
+
*(out++) = 0;
|
|
209550
|
+
}
|
|
209551
|
+
|
|
209552
|
+
void __fastunpack0(const uint32_t *__restrict, uint32_t *__restrict out) {
|
|
209553
|
+
for (uint32_t i = 0; i < 32; ++i)
|
|
209554
|
+
*(out++) = 0;
|
|
209555
|
+
}
|
|
209556
|
+
|
|
209557
|
+
void __fastunpack0(const uint32_t *__restrict, uint64_t *__restrict out) {
|
|
209558
|
+
for (uint32_t i = 0; i < 32; ++i)
|
|
209559
|
+
*(out++) = 0;
|
|
209560
|
+
}
|
|
209561
|
+
|
|
209562
|
+
void __fastpack0(const uint8_t *__restrict, uint8_t *__restrict) {
|
|
209563
|
+
}
|
|
209564
|
+
void __fastpack0(const uint16_t *__restrict, uint16_t *__restrict) {
|
|
209565
|
+
}
|
|
209566
|
+
void __fastpack0(const uint32_t *__restrict, uint32_t *__restrict) {
|
|
209567
|
+
}
|
|
209568
|
+
void __fastpack0(const uint64_t *__restrict, uint32_t *__restrict) {
|
|
209569
|
+
}
|
|
209570
|
+
|
|
209571
|
+
// fastunpack for 8 bits
|
|
209572
|
+
void __fastunpack1(const uint8_t *__restrict in, uint8_t *__restrict out) {
|
|
209573
|
+
Unroller8<1>::Unpack(in, out);
|
|
209574
|
+
}
|
|
209575
|
+
|
|
209576
|
+
void __fastunpack2(const uint8_t *__restrict in, uint8_t *__restrict out) {
|
|
209577
|
+
Unroller8<2>::Unpack(in, out);
|
|
209578
|
+
}
|
|
209579
|
+
|
|
209580
|
+
void __fastunpack3(const uint8_t *__restrict in, uint8_t *__restrict out) {
|
|
209581
|
+
Unroller8<3>::Unpack(in, out);
|
|
209582
|
+
}
|
|
209583
|
+
|
|
209584
|
+
void __fastunpack4(const uint8_t *__restrict in, uint8_t *__restrict out) {
|
|
209585
|
+
for (uint8_t outer = 0; outer < 4; ++outer) {
|
|
209586
|
+
for (uint8_t inwordpointer = 0; inwordpointer < 8; inwordpointer += 4)
|
|
209587
|
+
*(out++) = ((*in) >> inwordpointer) % (1U << 4);
|
|
209588
|
+
++in;
|
|
209589
|
+
}
|
|
209590
|
+
}
|
|
209591
|
+
|
|
209592
|
+
void __fastunpack5(const uint8_t *__restrict in, uint8_t *__restrict out) {
|
|
209593
|
+
Unroller8<5>::Unpack(in, out);
|
|
209594
|
+
}
|
|
209595
|
+
|
|
209596
|
+
void __fastunpack6(const uint8_t *__restrict in, uint8_t *__restrict out) {
|
|
209597
|
+
Unroller8<6>::Unpack(in, out);
|
|
209598
|
+
}
|
|
209599
|
+
|
|
209600
|
+
void __fastunpack7(const uint8_t *__restrict in, uint8_t *__restrict out) {
|
|
209601
|
+
Unroller8<7>::Unpack(in, out);
|
|
209602
|
+
}
|
|
209603
|
+
|
|
209604
|
+
void __fastunpack8(const uint8_t *__restrict in, uint8_t *__restrict out) {
|
|
209605
|
+
for (int k = 0; k < 8; ++k)
|
|
209606
|
+
out[k] = in[k];
|
|
209607
|
+
}
|
|
209608
|
+
|
|
209609
|
+
|
|
209610
|
+
// fastunpack for 16 bits
|
|
209611
|
+
void __fastunpack1(const uint16_t *__restrict in, uint16_t *__restrict out) {
|
|
209612
|
+
Unroller16<1>::Unpack(in, out);
|
|
209613
|
+
}
|
|
209614
|
+
|
|
209615
|
+
void __fastunpack2(const uint16_t *__restrict in, uint16_t *__restrict out) {
|
|
209616
|
+
Unroller16<2>::Unpack(in, out);
|
|
209617
|
+
}
|
|
209618
|
+
|
|
209619
|
+
void __fastunpack3(const uint16_t *__restrict in, uint16_t *__restrict out) {
|
|
209620
|
+
Unroller16<3>::Unpack(in, out);
|
|
209621
|
+
}
|
|
209622
|
+
|
|
209623
|
+
void __fastunpack4(const uint16_t *__restrict in, uint16_t *__restrict out) {
|
|
209624
|
+
for (uint16_t outer = 0; outer < 4; ++outer) {
|
|
209625
|
+
for (uint16_t inwordpointer = 0; inwordpointer < 16; inwordpointer += 4)
|
|
209626
|
+
*(out++) = ((*in) >> inwordpointer) % (1U << 4);
|
|
209627
|
+
++in;
|
|
209628
|
+
}
|
|
209629
|
+
}
|
|
209630
|
+
|
|
209631
|
+
void __fastunpack5(const uint16_t *__restrict in, uint16_t *__restrict out) {
|
|
209632
|
+
Unroller16<5>::Unpack(in, out);
|
|
209633
|
+
}
|
|
209634
|
+
|
|
209635
|
+
void __fastunpack6(const uint16_t *__restrict in, uint16_t *__restrict out) {
|
|
209636
|
+
Unroller16<6>::Unpack(in, out);
|
|
209637
|
+
}
|
|
209638
|
+
|
|
209639
|
+
void __fastunpack7(const uint16_t *__restrict in, uint16_t *__restrict out) {
|
|
209640
|
+
Unroller16<7>::Unpack(in, out);
|
|
209641
|
+
}
|
|
209642
|
+
|
|
209643
|
+
void __fastunpack8(const uint16_t *__restrict in, uint16_t *__restrict out) {
|
|
209644
|
+
for (uint16_t outer = 0; outer < 8; ++outer) {
|
|
209645
|
+
for (uint16_t inwordpointer = 0; inwordpointer < 16; inwordpointer += 8)
|
|
209646
|
+
*(out++) = ((*in) >> inwordpointer) % (1U << 8);
|
|
209647
|
+
++in;
|
|
209648
|
+
}
|
|
209649
|
+
}
|
|
209650
|
+
|
|
209651
|
+
void __fastunpack9(const uint16_t *__restrict in, uint16_t *__restrict out) {
|
|
209652
|
+
Unroller16<9>::Unpack(in, out);
|
|
209653
|
+
}
|
|
209654
|
+
|
|
209655
|
+
void __fastunpack10(const uint16_t *__restrict in, uint16_t *__restrict out) {
|
|
209656
|
+
Unroller16<10>::Unpack(in, out);
|
|
209657
|
+
}
|
|
209658
|
+
|
|
209659
|
+
void __fastunpack11(const uint16_t *__restrict in, uint16_t *__restrict out) {
|
|
209660
|
+
Unroller16<11>::Unpack(in, out);
|
|
209661
|
+
}
|
|
209662
|
+
|
|
209663
|
+
void __fastunpack12(const uint16_t *__restrict in, uint16_t *__restrict out) {
|
|
209664
|
+
Unroller16<12>::Unpack(in, out);
|
|
209665
|
+
}
|
|
209666
|
+
|
|
209667
|
+
void __fastunpack13(const uint16_t *__restrict in, uint16_t *__restrict out) {
|
|
209668
|
+
Unroller16<13>::Unpack(in, out);
|
|
209669
|
+
}
|
|
209670
|
+
|
|
209671
|
+
void __fastunpack14(const uint16_t *__restrict in, uint16_t *__restrict out) {
|
|
209672
|
+
Unroller16<14>::Unpack(in, out);
|
|
209673
|
+
}
|
|
209674
|
+
|
|
209675
|
+
void __fastunpack15(const uint16_t *__restrict in, uint16_t *__restrict out) {
|
|
209676
|
+
Unroller16<15>::Unpack(in, out);
|
|
209677
|
+
}
|
|
209678
|
+
|
|
209679
|
+
void __fastunpack16(const uint16_t *__restrict in, uint16_t *__restrict out) {
|
|
209680
|
+
for (int k = 0; k < 16; ++k)
|
|
209681
|
+
out[k] = in[k];
|
|
209682
|
+
}
|
|
209683
|
+
|
|
209684
|
+
// fastunpack for 32 bits
|
|
209685
|
+
void __fastunpack1(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
209686
|
+
Unroller<1>::Unpack(in, out);
|
|
209687
|
+
}
|
|
209688
|
+
|
|
209689
|
+
void __fastunpack2(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
209690
|
+
Unroller<2>::Unpack(in, out);
|
|
209691
|
+
}
|
|
209692
|
+
|
|
209693
|
+
void __fastunpack3(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
209694
|
+
Unroller<3>::Unpack(in, out);
|
|
209695
|
+
}
|
|
209696
|
+
|
|
209697
|
+
void __fastunpack4(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
209698
|
+
for (uint32_t outer = 0; outer < 4; ++outer) {
|
|
209699
|
+
for (uint32_t inwordpointer = 0; inwordpointer < 32; inwordpointer += 4)
|
|
209700
|
+
*(out++) = ((*in) >> inwordpointer) % (1U << 4);
|
|
209701
|
+
++in;
|
|
209702
|
+
}
|
|
209703
|
+
}
|
|
209704
|
+
|
|
209705
|
+
void __fastunpack5(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
209706
|
+
Unroller<5>::Unpack(in, out);
|
|
209707
|
+
}
|
|
209708
|
+
|
|
209709
|
+
void __fastunpack6(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
209710
|
+
Unroller<6>::Unpack(in, out);
|
|
209711
|
+
}
|
|
209712
|
+
|
|
209713
|
+
void __fastunpack7(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
209714
|
+
Unroller<7>::Unpack(in, out);
|
|
209715
|
+
}
|
|
209716
|
+
|
|
209717
|
+
void __fastunpack8(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
209718
|
+
for (uint32_t outer = 0; outer < 8; ++outer) {
|
|
209719
|
+
for (uint32_t inwordpointer = 0; inwordpointer < 32; inwordpointer += 8)
|
|
209720
|
+
*(out++) = ((*in) >> inwordpointer) % (1U << 8);
|
|
209721
|
+
++in;
|
|
209722
|
+
}
|
|
209723
|
+
}
|
|
209724
|
+
|
|
209725
|
+
void __fastunpack9(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
209726
|
+
Unroller<9>::Unpack(in, out);
|
|
209727
|
+
}
|
|
209728
|
+
|
|
209729
|
+
void __fastunpack10(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
209730
|
+
Unroller<10>::Unpack(in, out);
|
|
209731
|
+
}
|
|
209732
|
+
|
|
209733
|
+
void __fastunpack11(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
209734
|
+
Unroller<11>::Unpack(in, out);
|
|
209735
|
+
}
|
|
209736
|
+
|
|
209737
|
+
void __fastunpack12(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
209738
|
+
Unroller<12>::Unpack(in, out);
|
|
209739
|
+
}
|
|
209740
|
+
|
|
209741
|
+
void __fastunpack13(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
209742
|
+
Unroller<13>::Unpack(in, out);
|
|
209743
|
+
}
|
|
209744
|
+
|
|
209745
|
+
void __fastunpack14(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
209746
|
+
Unroller<14>::Unpack(in, out);
|
|
209747
|
+
}
|
|
209748
|
+
|
|
209749
|
+
void __fastunpack15(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
209750
|
+
Unroller<15>::Unpack(in, out);
|
|
209751
|
+
}
|
|
209752
|
+
|
|
209753
|
+
void __fastunpack16(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
209754
|
+
for (uint32_t outer = 0; outer < 16; ++outer) {
|
|
209755
|
+
for (uint32_t inwordpointer = 0; inwordpointer < 32; inwordpointer += 16)
|
|
209756
|
+
*(out++) = ((*in) >> inwordpointer) % (1U << 16);
|
|
209757
|
+
++in;
|
|
209758
|
+
}
|
|
209759
|
+
}
|
|
209760
|
+
|
|
209761
|
+
void __fastunpack17(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
209762
|
+
Unroller<17>::Unpack(in, out);
|
|
209763
|
+
}
|
|
209764
|
+
|
|
209765
|
+
void __fastunpack18(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
209766
|
+
Unroller<18>::Unpack(in, out);
|
|
209767
|
+
}
|
|
209768
|
+
|
|
209769
|
+
void __fastunpack19(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
209770
|
+
Unroller<19>::Unpack(in, out);
|
|
209771
|
+
}
|
|
209772
|
+
|
|
209773
|
+
void __fastunpack20(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
209774
|
+
Unroller<20>::Unpack(in, out);
|
|
209775
|
+
}
|
|
209776
|
+
|
|
209777
|
+
void __fastunpack21(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
209778
|
+
Unroller<21>::Unpack(in, out);
|
|
209779
|
+
}
|
|
209780
|
+
|
|
209781
|
+
void __fastunpack22(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
209782
|
+
Unroller<22>::Unpack(in, out);
|
|
209783
|
+
}
|
|
209784
|
+
|
|
209785
|
+
void __fastunpack23(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
209786
|
+
Unroller<23>::Unpack(in, out);
|
|
209787
|
+
}
|
|
209788
|
+
|
|
209789
|
+
void __fastunpack24(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
209790
|
+
Unroller<24>::Unpack(in, out);
|
|
209791
|
+
}
|
|
209792
|
+
|
|
209793
|
+
void __fastunpack25(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
209794
|
+
Unroller<25>::Unpack(in, out);
|
|
209795
|
+
}
|
|
209796
|
+
|
|
209797
|
+
void __fastunpack26(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
209798
|
+
Unroller<26>::Unpack(in, out);
|
|
209799
|
+
}
|
|
209800
|
+
|
|
209801
|
+
void __fastunpack27(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
209802
|
+
Unroller<27>::Unpack(in, out);
|
|
209803
|
+
}
|
|
209804
|
+
|
|
209805
|
+
void __fastunpack28(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
209806
|
+
Unroller<28>::Unpack(in, out);
|
|
209807
|
+
}
|
|
209808
|
+
|
|
209809
|
+
void __fastunpack29(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
209810
|
+
Unroller<29>::Unpack(in, out);
|
|
209811
|
+
}
|
|
209812
|
+
|
|
209813
|
+
void __fastunpack30(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
209814
|
+
Unroller<30>::Unpack(in, out);
|
|
209815
|
+
}
|
|
209816
|
+
|
|
209817
|
+
void __fastunpack31(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
209818
|
+
Unroller<31>::Unpack(in, out);
|
|
209819
|
+
}
|
|
209820
|
+
|
|
209821
|
+
void __fastunpack32(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
209822
|
+
for (int k = 0; k < 32; ++k)
|
|
209823
|
+
out[k] = in[k];
|
|
209824
|
+
}
|
|
209825
|
+
|
|
209826
|
+
// fastupack for 64 bits
|
|
209827
|
+
void __fastunpack1(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209828
|
+
Unroller<1>::Unpack(in, out);
|
|
209829
|
+
}
|
|
209830
|
+
|
|
209831
|
+
void __fastunpack2(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209832
|
+
Unroller<2>::Unpack(in, out);
|
|
209833
|
+
}
|
|
209834
|
+
|
|
209835
|
+
void __fastunpack3(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209836
|
+
Unroller<3>::Unpack(in, out);
|
|
209837
|
+
}
|
|
209838
|
+
|
|
209839
|
+
void __fastunpack4(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209840
|
+
for (uint32_t outer = 0; outer < 4; ++outer) {
|
|
209841
|
+
for (uint32_t inwordpointer = 0; inwordpointer < 32; inwordpointer += 4)
|
|
209842
|
+
*(out++) = ((*in) >> inwordpointer) % (1U << 4);
|
|
209843
|
+
++in;
|
|
209844
|
+
}
|
|
209845
|
+
}
|
|
209846
|
+
|
|
209847
|
+
void __fastunpack5(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209848
|
+
Unroller<5>::Unpack(in, out);
|
|
209849
|
+
}
|
|
209850
|
+
|
|
209851
|
+
void __fastunpack6(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209852
|
+
Unroller<6>::Unpack(in, out);
|
|
209853
|
+
}
|
|
209854
|
+
|
|
209855
|
+
void __fastunpack7(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209856
|
+
Unroller<7>::Unpack(in, out);
|
|
209857
|
+
}
|
|
209858
|
+
|
|
209859
|
+
void __fastunpack8(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209860
|
+
for (uint32_t outer = 0; outer < 8; ++outer) {
|
|
209861
|
+
for (uint32_t inwordpointer = 0; inwordpointer < 32; inwordpointer += 8) {
|
|
209862
|
+
*(out++) = ((*in) >> inwordpointer) % (1U << 8);
|
|
209863
|
+
}
|
|
209864
|
+
++in;
|
|
209865
|
+
}
|
|
209866
|
+
}
|
|
209867
|
+
|
|
209868
|
+
void __fastunpack9(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209869
|
+
Unroller<9>::Unpack(in, out);
|
|
209870
|
+
}
|
|
209871
|
+
|
|
209872
|
+
void __fastunpack10(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209873
|
+
Unroller<10>::Unpack(in, out);
|
|
209874
|
+
}
|
|
209875
|
+
|
|
209876
|
+
void __fastunpack11(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209877
|
+
Unroller<11>::Unpack(in, out);
|
|
209878
|
+
}
|
|
209879
|
+
|
|
209880
|
+
void __fastunpack12(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209881
|
+
Unroller<12>::Unpack(in, out);
|
|
209882
|
+
}
|
|
209883
|
+
|
|
209884
|
+
void __fastunpack13(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209885
|
+
Unroller<13>::Unpack(in, out);
|
|
209886
|
+
}
|
|
209887
|
+
|
|
209888
|
+
void __fastunpack14(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209889
|
+
Unroller<14>::Unpack(in, out);
|
|
209890
|
+
}
|
|
209891
|
+
|
|
209892
|
+
void __fastunpack15(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209893
|
+
Unroller<15>::Unpack(in, out);
|
|
209894
|
+
}
|
|
209895
|
+
|
|
209896
|
+
void __fastunpack16(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209897
|
+
for (uint32_t outer = 0; outer < 16; ++outer) {
|
|
209898
|
+
for (uint32_t inwordpointer = 0; inwordpointer < 32; inwordpointer += 16)
|
|
209899
|
+
*(out++) = ((*in) >> inwordpointer) % (1U << 16);
|
|
209900
|
+
++in;
|
|
209901
|
+
}
|
|
209902
|
+
}
|
|
209903
|
+
|
|
209904
|
+
void __fastunpack17(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209905
|
+
Unroller<17>::Unpack(in, out);
|
|
209906
|
+
}
|
|
209907
|
+
|
|
209908
|
+
void __fastunpack18(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209909
|
+
Unroller<18>::Unpack(in, out);
|
|
209910
|
+
}
|
|
209911
|
+
|
|
209912
|
+
void __fastunpack19(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209913
|
+
Unroller<19>::Unpack(in, out);
|
|
209914
|
+
}
|
|
209915
|
+
|
|
209916
|
+
void __fastunpack20(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209917
|
+
Unroller<20>::Unpack(in, out);
|
|
209918
|
+
}
|
|
209919
|
+
|
|
209920
|
+
void __fastunpack21(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209921
|
+
Unroller<21>::Unpack(in, out);
|
|
209922
|
+
}
|
|
209923
|
+
|
|
209924
|
+
void __fastunpack22(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209925
|
+
Unroller<22>::Unpack(in, out);
|
|
209926
|
+
}
|
|
209927
|
+
|
|
209928
|
+
void __fastunpack23(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209929
|
+
Unroller<23>::Unpack(in, out);
|
|
209930
|
+
}
|
|
209931
|
+
|
|
209932
|
+
void __fastunpack24(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209933
|
+
Unroller<24>::Unpack(in, out);
|
|
209934
|
+
}
|
|
209935
|
+
|
|
209936
|
+
void __fastunpack25(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209937
|
+
Unroller<25>::Unpack(in, out);
|
|
209938
|
+
}
|
|
209939
|
+
|
|
209940
|
+
void __fastunpack26(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209941
|
+
Unroller<26>::Unpack(in, out);
|
|
209942
|
+
}
|
|
209943
|
+
|
|
209944
|
+
void __fastunpack27(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209945
|
+
Unroller<27>::Unpack(in, out);
|
|
209946
|
+
}
|
|
209947
|
+
|
|
209948
|
+
void __fastunpack28(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209949
|
+
Unroller<28>::Unpack(in, out);
|
|
209950
|
+
}
|
|
209951
|
+
|
|
209952
|
+
void __fastunpack29(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209953
|
+
Unroller<29>::Unpack(in, out);
|
|
209954
|
+
}
|
|
209955
|
+
|
|
209956
|
+
void __fastunpack30(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209957
|
+
Unroller<30>::Unpack(in, out);
|
|
209958
|
+
}
|
|
209959
|
+
|
|
209960
|
+
void __fastunpack31(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209961
|
+
Unroller<31>::Unpack(in, out);
|
|
209962
|
+
}
|
|
209963
|
+
|
|
209964
|
+
void __fastunpack32(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209965
|
+
for (int k = 0; k < 32; ++k)
|
|
209966
|
+
out[k] = in[k];
|
|
209967
|
+
}
|
|
209968
|
+
|
|
209969
|
+
void __fastunpack33(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209970
|
+
Unroller<33>::Unpack(in, out);
|
|
209971
|
+
}
|
|
209972
|
+
|
|
209973
|
+
void __fastunpack34(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209974
|
+
Unroller<34>::Unpack(in, out);
|
|
209975
|
+
}
|
|
209976
|
+
|
|
209977
|
+
void __fastunpack35(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209978
|
+
Unroller<35>::Unpack(in, out);
|
|
209979
|
+
}
|
|
209980
|
+
|
|
209981
|
+
void __fastunpack36(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209982
|
+
Unroller<36>::Unpack(in, out);
|
|
209983
|
+
}
|
|
209984
|
+
|
|
209985
|
+
void __fastunpack37(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209986
|
+
Unroller<37>::Unpack(in, out);
|
|
209987
|
+
}
|
|
209988
|
+
|
|
209989
|
+
void __fastunpack38(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209990
|
+
Unroller<38>::Unpack(in, out);
|
|
209991
|
+
}
|
|
209992
|
+
|
|
209993
|
+
void __fastunpack39(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209994
|
+
Unroller<39>::Unpack(in, out);
|
|
209995
|
+
}
|
|
209996
|
+
|
|
209997
|
+
void __fastunpack40(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
209998
|
+
Unroller<40>::Unpack(in, out);
|
|
209999
|
+
}
|
|
210000
|
+
|
|
210001
|
+
void __fastunpack41(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
210002
|
+
Unroller<41>::Unpack(in, out);
|
|
210003
|
+
}
|
|
210004
|
+
|
|
210005
|
+
void __fastunpack42(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
210006
|
+
Unroller<42>::Unpack(in, out);
|
|
210007
|
+
}
|
|
210008
|
+
|
|
210009
|
+
void __fastunpack43(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
210010
|
+
Unroller<43>::Unpack(in, out);
|
|
210011
|
+
}
|
|
210012
|
+
|
|
210013
|
+
void __fastunpack44(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
210014
|
+
Unroller<44>::Unpack(in, out);
|
|
210015
|
+
}
|
|
210016
|
+
|
|
210017
|
+
void __fastunpack45(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
210018
|
+
Unroller<45>::Unpack(in, out);
|
|
210019
|
+
}
|
|
210020
|
+
|
|
210021
|
+
void __fastunpack46(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
210022
|
+
Unroller<46>::Unpack(in, out);
|
|
210023
|
+
}
|
|
210024
|
+
|
|
210025
|
+
void __fastunpack47(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
210026
|
+
Unroller<47>::Unpack(in, out);
|
|
210027
|
+
}
|
|
210028
|
+
|
|
210029
|
+
void __fastunpack48(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
210030
|
+
Unroller<48>::Unpack(in, out);
|
|
210031
|
+
}
|
|
210032
|
+
|
|
210033
|
+
void __fastunpack49(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
210034
|
+
Unroller<49>::Unpack(in, out);
|
|
210035
|
+
}
|
|
210036
|
+
|
|
210037
|
+
void __fastunpack50(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
210038
|
+
Unroller<50>::Unpack(in, out);
|
|
210039
|
+
}
|
|
210040
|
+
|
|
210041
|
+
void __fastunpack51(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
210042
|
+
Unroller<51>::Unpack(in, out);
|
|
210043
|
+
}
|
|
210044
|
+
|
|
210045
|
+
void __fastunpack52(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
210046
|
+
Unroller<52>::Unpack(in, out);
|
|
210047
|
+
}
|
|
210048
|
+
|
|
210049
|
+
void __fastunpack53(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
210050
|
+
Unroller<53>::Unpack(in, out);
|
|
210051
|
+
}
|
|
210052
|
+
|
|
210053
|
+
void __fastunpack54(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
210054
|
+
Unroller<54>::Unpack(in, out);
|
|
210055
|
+
}
|
|
210056
|
+
|
|
210057
|
+
void __fastunpack55(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
210058
|
+
Unroller<55>::Unpack(in, out);
|
|
210059
|
+
}
|
|
210060
|
+
|
|
210061
|
+
void __fastunpack56(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
210062
|
+
Unroller<56>::Unpack(in, out);
|
|
210063
|
+
}
|
|
210064
|
+
|
|
210065
|
+
void __fastunpack57(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
210066
|
+
Unroller<57>::Unpack(in, out);
|
|
210067
|
+
}
|
|
210068
|
+
|
|
210069
|
+
void __fastunpack58(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
210070
|
+
Unroller<58>::Unpack(in, out);
|
|
210071
|
+
}
|
|
210072
|
+
|
|
210073
|
+
void __fastunpack59(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
210074
|
+
Unroller<59>::Unpack(in, out);
|
|
210075
|
+
}
|
|
210076
|
+
|
|
210077
|
+
void __fastunpack60(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
210078
|
+
Unroller<60>::Unpack(in, out);
|
|
210079
|
+
}
|
|
210080
|
+
|
|
210081
|
+
void __fastunpack61(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
210082
|
+
Unroller<61>::Unpack(in, out);
|
|
210083
|
+
}
|
|
210084
|
+
|
|
210085
|
+
void __fastunpack62(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
210086
|
+
Unroller<62>::Unpack(in, out);
|
|
210087
|
+
}
|
|
210088
|
+
|
|
210089
|
+
void __fastunpack63(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
210090
|
+
Unroller<63>::Unpack(in, out);
|
|
210091
|
+
}
|
|
210092
|
+
|
|
210093
|
+
void __fastunpack64(const uint32_t *__restrict in, uint64_t *__restrict out) {
|
|
210094
|
+
for (int k = 0; k < 32; ++k) {
|
|
210095
|
+
out[k] = in[k * 2];
|
|
210096
|
+
out[k] |= static_cast<uint64_t>(in[k * 2 + 1]) << 32;
|
|
210097
|
+
}
|
|
210098
|
+
}
|
|
210099
|
+
|
|
210100
|
+
// fastpack for 8 bits
|
|
210101
|
+
|
|
210102
|
+
void __fastpack1(const uint8_t *__restrict in, uint8_t *__restrict out) {
|
|
210103
|
+
Unroller8<1>::Pack(in, out);
|
|
210104
|
+
}
|
|
210105
|
+
|
|
210106
|
+
void __fastpack2(const uint8_t *__restrict in, uint8_t *__restrict out) {
|
|
210107
|
+
Unroller8<2>::Pack(in, out);
|
|
210108
|
+
}
|
|
210109
|
+
|
|
210110
|
+
void __fastpack3(const uint8_t *__restrict in, uint8_t *__restrict out) {
|
|
210111
|
+
Unroller8<3>::Pack(in, out);
|
|
210112
|
+
}
|
|
210113
|
+
|
|
210114
|
+
void __fastpack4(const uint8_t *__restrict in, uint8_t *__restrict out) {
|
|
210115
|
+
Unroller8<4>::Pack(in, out);
|
|
210116
|
+
}
|
|
210117
|
+
|
|
210118
|
+
void __fastpack5(const uint8_t *__restrict in, uint8_t *__restrict out) {
|
|
210119
|
+
Unroller8<5>::Pack(in, out);
|
|
210120
|
+
}
|
|
210121
|
+
|
|
210122
|
+
void __fastpack6(const uint8_t *__restrict in, uint8_t *__restrict out) {
|
|
210123
|
+
Unroller8<6>::Pack(in, out);
|
|
210124
|
+
}
|
|
210125
|
+
|
|
210126
|
+
void __fastpack7(const uint8_t *__restrict in, uint8_t *__restrict out) {
|
|
210127
|
+
Unroller8<7>::Pack(in, out);
|
|
210128
|
+
}
|
|
210129
|
+
|
|
210130
|
+
void __fastpack8(const uint8_t *__restrict in, uint8_t *__restrict out) {
|
|
210131
|
+
for (int k = 0; k < 8; ++k)
|
|
210132
|
+
out[k] = in[k];
|
|
210133
|
+
}
|
|
210134
|
+
|
|
210135
|
+
// fastpack for 16 bits
|
|
210136
|
+
|
|
210137
|
+
void __fastpack1(const uint16_t *__restrict in, uint16_t *__restrict out) {
|
|
210138
|
+
Unroller16<1>::Pack(in, out);
|
|
210139
|
+
}
|
|
210140
|
+
|
|
210141
|
+
void __fastpack2(const uint16_t *__restrict in, uint16_t *__restrict out) {
|
|
210142
|
+
Unroller16<2>::Pack(in, out);
|
|
210143
|
+
}
|
|
210144
|
+
|
|
210145
|
+
void __fastpack3(const uint16_t *__restrict in, uint16_t *__restrict out) {
|
|
210146
|
+
Unroller16<3>::Pack(in, out);
|
|
210147
|
+
}
|
|
210148
|
+
|
|
210149
|
+
void __fastpack4(const uint16_t *__restrict in, uint16_t *__restrict out) {
|
|
210150
|
+
Unroller16<4>::Pack(in, out);
|
|
210151
|
+
}
|
|
210152
|
+
|
|
210153
|
+
void __fastpack5(const uint16_t *__restrict in, uint16_t *__restrict out) {
|
|
210154
|
+
Unroller16<5>::Pack(in, out);
|
|
210155
|
+
}
|
|
210156
|
+
|
|
210157
|
+
void __fastpack6(const uint16_t *__restrict in, uint16_t *__restrict out) {
|
|
210158
|
+
Unroller16<6>::Pack(in, out);
|
|
210159
|
+
}
|
|
210160
|
+
|
|
210161
|
+
void __fastpack7(const uint16_t *__restrict in, uint16_t *__restrict out) {
|
|
210162
|
+
Unroller16<7>::Pack(in, out);
|
|
210163
|
+
}
|
|
210164
|
+
|
|
210165
|
+
void __fastpack8(const uint16_t *__restrict in, uint16_t *__restrict out) {
|
|
210166
|
+
Unroller16<8>::Pack(in, out);
|
|
210167
|
+
}
|
|
210168
|
+
|
|
210169
|
+
void __fastpack9(const uint16_t *__restrict in, uint16_t *__restrict out) {
|
|
210170
|
+
Unroller16<9>::Pack(in, out);
|
|
210171
|
+
}
|
|
210172
|
+
|
|
210173
|
+
void __fastpack10(const uint16_t *__restrict in, uint16_t *__restrict out) {
|
|
210174
|
+
Unroller16<10>::Pack(in, out);
|
|
210175
|
+
}
|
|
210176
|
+
|
|
210177
|
+
void __fastpack11(const uint16_t *__restrict in, uint16_t *__restrict out) {
|
|
210178
|
+
Unroller16<11>::Pack(in, out);
|
|
210179
|
+
}
|
|
210180
|
+
|
|
210181
|
+
void __fastpack12(const uint16_t *__restrict in, uint16_t *__restrict out) {
|
|
210182
|
+
Unroller16<12>::Pack(in, out);
|
|
210183
|
+
}
|
|
210184
|
+
|
|
210185
|
+
void __fastpack13(const uint16_t *__restrict in, uint16_t *__restrict out) {
|
|
210186
|
+
Unroller16<13>::Pack(in, out);
|
|
210187
|
+
}
|
|
210188
|
+
|
|
210189
|
+
void __fastpack14(const uint16_t *__restrict in, uint16_t *__restrict out) {
|
|
210190
|
+
Unroller16<14>::Pack(in, out);
|
|
210191
|
+
}
|
|
210192
|
+
|
|
210193
|
+
void __fastpack15(const uint16_t *__restrict in, uint16_t *__restrict out) {
|
|
210194
|
+
Unroller16<15>::Pack(in, out);
|
|
210195
|
+
}
|
|
210196
|
+
|
|
210197
|
+
void __fastpack16(const uint16_t *__restrict in, uint16_t *__restrict out) {
|
|
210198
|
+
for (int k = 0; k < 16; ++k)
|
|
210199
|
+
out[k] = in[k];
|
|
210200
|
+
}
|
|
210201
|
+
|
|
210202
|
+
|
|
210203
|
+
// fastpack for 32 bits
|
|
210204
|
+
|
|
210205
|
+
void __fastpack1(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
210206
|
+
Unroller<1>::Pack(in, out);
|
|
210207
|
+
}
|
|
210208
|
+
|
|
210209
|
+
void __fastpack2(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
210210
|
+
Unroller<2>::Pack(in, out);
|
|
210211
|
+
}
|
|
210212
|
+
|
|
210213
|
+
void __fastpack3(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
210214
|
+
Unroller<3>::Pack(in, out);
|
|
210215
|
+
}
|
|
210216
|
+
|
|
210217
|
+
void __fastpack4(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
210218
|
+
Unroller<4>::Pack(in, out);
|
|
210219
|
+
}
|
|
210220
|
+
|
|
210221
|
+
void __fastpack5(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
210222
|
+
Unroller<5>::Pack(in, out);
|
|
210223
|
+
}
|
|
210224
|
+
|
|
210225
|
+
void __fastpack6(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
210226
|
+
Unroller<6>::Pack(in, out);
|
|
210227
|
+
}
|
|
210228
|
+
|
|
210229
|
+
void __fastpack7(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
210230
|
+
Unroller<7>::Pack(in, out);
|
|
210231
|
+
}
|
|
210232
|
+
|
|
210233
|
+
void __fastpack8(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
210234
|
+
Unroller<8>::Pack(in, out);
|
|
210235
|
+
}
|
|
210236
|
+
|
|
210237
|
+
void __fastpack9(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
210238
|
+
Unroller<9>::Pack(in, out);
|
|
210239
|
+
}
|
|
210240
|
+
|
|
210241
|
+
void __fastpack10(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
210242
|
+
Unroller<10>::Pack(in, out);
|
|
210243
|
+
}
|
|
210244
|
+
|
|
210245
|
+
void __fastpack11(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
210246
|
+
Unroller<11>::Pack(in, out);
|
|
210247
|
+
}
|
|
210248
|
+
|
|
210249
|
+
void __fastpack12(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
210250
|
+
Unroller<12>::Pack(in, out);
|
|
210251
|
+
}
|
|
210252
|
+
|
|
210253
|
+
void __fastpack13(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
210254
|
+
Unroller<13>::Pack(in, out);
|
|
210255
|
+
}
|
|
210256
|
+
|
|
210257
|
+
void __fastpack14(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
210258
|
+
Unroller<14>::Pack(in, out);
|
|
210259
|
+
}
|
|
210260
|
+
|
|
210261
|
+
void __fastpack15(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
210262
|
+
Unroller<15>::Pack(in, out);
|
|
210263
|
+
}
|
|
210264
|
+
|
|
210265
|
+
void __fastpack16(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
210266
|
+
Unroller<16>::Pack(in, out);
|
|
210267
|
+
}
|
|
210268
|
+
|
|
210269
|
+
void __fastpack17(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
210270
|
+
Unroller<17>::Pack(in, out);
|
|
210271
|
+
}
|
|
210272
|
+
|
|
210273
|
+
void __fastpack18(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
210274
|
+
Unroller<18>::Pack(in, out);
|
|
210275
|
+
}
|
|
210276
|
+
|
|
210277
|
+
void __fastpack19(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
210278
|
+
Unroller<19>::Pack(in, out);
|
|
210279
|
+
}
|
|
210280
|
+
|
|
210281
|
+
void __fastpack20(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
210282
|
+
Unroller<20>::Pack(in, out);
|
|
210283
|
+
}
|
|
210284
|
+
|
|
210285
|
+
void __fastpack21(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
210286
|
+
Unroller<21>::Pack(in, out);
|
|
210287
|
+
}
|
|
210288
|
+
|
|
210289
|
+
void __fastpack22(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
210290
|
+
Unroller<22>::Pack(in, out);
|
|
210291
|
+
}
|
|
210292
|
+
|
|
210293
|
+
void __fastpack23(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
210294
|
+
Unroller<23>::Pack(in, out);
|
|
210295
|
+
}
|
|
210296
|
+
|
|
210297
|
+
void __fastpack24(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
210298
|
+
Unroller<24>::Pack(in, out);
|
|
210299
|
+
}
|
|
210300
|
+
|
|
210301
|
+
void __fastpack25(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
210302
|
+
Unroller<25>::Pack(in, out);
|
|
210303
|
+
}
|
|
210304
|
+
|
|
210305
|
+
void __fastpack26(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
210306
|
+
Unroller<26>::Pack(in, out);
|
|
210307
|
+
}
|
|
210308
|
+
|
|
210309
|
+
void __fastpack27(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
210310
|
+
Unroller<27>::Pack(in, out);
|
|
210311
|
+
}
|
|
210312
|
+
|
|
210313
|
+
void __fastpack28(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
210314
|
+
Unroller<28>::Pack(in, out);
|
|
210315
|
+
}
|
|
210316
|
+
|
|
210317
|
+
void __fastpack29(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
210318
|
+
Unroller<29>::Pack(in, out);
|
|
210319
|
+
}
|
|
210320
|
+
|
|
210321
|
+
void __fastpack30(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
210322
|
+
Unroller<30>::Pack(in, out);
|
|
210323
|
+
}
|
|
210324
|
+
|
|
210325
|
+
void __fastpack31(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
210326
|
+
Unroller<31>::Pack(in, out);
|
|
210327
|
+
}
|
|
210328
|
+
|
|
210329
|
+
void __fastpack32(const uint32_t *__restrict in, uint32_t *__restrict out) {
|
|
210330
|
+
for (int k = 0; k < 32; ++k)
|
|
210331
|
+
out[k] = in[k];
|
|
210332
|
+
}
|
|
210333
|
+
|
|
210334
|
+
// fastpack for 64 bits
|
|
210335
|
+
|
|
210336
|
+
void __fastpack1(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210337
|
+
Unroller<1>::Pack(in, out);
|
|
210338
|
+
}
|
|
210339
|
+
|
|
210340
|
+
void __fastpack2(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210341
|
+
Unroller<2>::Pack(in, out);
|
|
210342
|
+
}
|
|
210343
|
+
|
|
210344
|
+
void __fastpack3(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210345
|
+
Unroller<3>::Pack(in, out);
|
|
210346
|
+
}
|
|
210347
|
+
|
|
210348
|
+
void __fastpack4(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210349
|
+
Unroller<4>::Pack(in, out);
|
|
210350
|
+
}
|
|
210351
|
+
|
|
210352
|
+
void __fastpack5(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210353
|
+
Unroller<5>::Pack(in, out);
|
|
210354
|
+
}
|
|
210355
|
+
|
|
210356
|
+
void __fastpack6(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210357
|
+
Unroller<6>::Pack(in, out);
|
|
210358
|
+
}
|
|
210359
|
+
|
|
210360
|
+
void __fastpack7(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210361
|
+
Unroller<7>::Pack(in, out);
|
|
210362
|
+
}
|
|
210363
|
+
|
|
210364
|
+
void __fastpack8(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210365
|
+
Unroller<8>::Pack(in, out);
|
|
210366
|
+
}
|
|
210367
|
+
|
|
210368
|
+
void __fastpack9(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210369
|
+
Unroller<9>::Pack(in, out);
|
|
210370
|
+
}
|
|
210371
|
+
|
|
210372
|
+
void __fastpack10(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210373
|
+
Unroller<10>::Pack(in, out);
|
|
210374
|
+
}
|
|
210375
|
+
|
|
210376
|
+
void __fastpack11(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210377
|
+
Unroller<11>::Pack(in, out);
|
|
210378
|
+
}
|
|
210379
|
+
|
|
210380
|
+
void __fastpack12(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210381
|
+
Unroller<12>::Pack(in, out);
|
|
210382
|
+
}
|
|
210383
|
+
|
|
210384
|
+
void __fastpack13(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210385
|
+
Unroller<13>::Pack(in, out);
|
|
210386
|
+
}
|
|
210387
|
+
|
|
210388
|
+
void __fastpack14(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210389
|
+
Unroller<14>::Pack(in, out);
|
|
210390
|
+
}
|
|
210391
|
+
|
|
210392
|
+
void __fastpack15(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210393
|
+
Unroller<15>::Pack(in, out);
|
|
210394
|
+
}
|
|
210395
|
+
|
|
210396
|
+
void __fastpack16(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210397
|
+
Unroller<16>::Pack(in, out);
|
|
210398
|
+
}
|
|
210399
|
+
|
|
210400
|
+
void __fastpack17(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210401
|
+
Unroller<17>::Pack(in, out);
|
|
210402
|
+
}
|
|
210403
|
+
|
|
210404
|
+
void __fastpack18(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210405
|
+
Unroller<18>::Pack(in, out);
|
|
210406
|
+
}
|
|
210407
|
+
|
|
210408
|
+
void __fastpack19(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210409
|
+
Unroller<19>::Pack(in, out);
|
|
210410
|
+
}
|
|
210411
|
+
|
|
210412
|
+
void __fastpack20(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210413
|
+
Unroller<20>::Pack(in, out);
|
|
210414
|
+
}
|
|
210415
|
+
|
|
210416
|
+
void __fastpack21(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210417
|
+
Unroller<21>::Pack(in, out);
|
|
210418
|
+
}
|
|
210419
|
+
|
|
210420
|
+
void __fastpack22(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210421
|
+
Unroller<22>::Pack(in, out);
|
|
210422
|
+
}
|
|
210423
|
+
|
|
210424
|
+
void __fastpack23(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210425
|
+
Unroller<23>::Pack(in, out);
|
|
210426
|
+
}
|
|
210427
|
+
|
|
210428
|
+
void __fastpack24(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210429
|
+
Unroller<24>::Pack(in, out);
|
|
210430
|
+
}
|
|
210431
|
+
|
|
210432
|
+
void __fastpack25(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210433
|
+
Unroller<25>::Pack(in, out);
|
|
210434
|
+
}
|
|
210435
|
+
|
|
210436
|
+
void __fastpack26(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210437
|
+
Unroller<26>::Pack(in, out);
|
|
210438
|
+
}
|
|
210439
|
+
|
|
210440
|
+
void __fastpack27(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210441
|
+
Unroller<27>::Pack(in, out);
|
|
210442
|
+
}
|
|
210443
|
+
|
|
210444
|
+
void __fastpack28(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210445
|
+
Unroller<28>::Pack(in, out);
|
|
210446
|
+
}
|
|
210447
|
+
|
|
210448
|
+
void __fastpack29(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210449
|
+
Unroller<29>::Pack(in, out);
|
|
210450
|
+
}
|
|
210451
|
+
|
|
210452
|
+
void __fastpack30(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210453
|
+
Unroller<30>::Pack(in, out);
|
|
210454
|
+
}
|
|
210455
|
+
|
|
210456
|
+
void __fastpack31(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210457
|
+
Unroller<31>::Pack(in, out);
|
|
210458
|
+
}
|
|
210459
|
+
|
|
210460
|
+
void __fastpack32(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210461
|
+
for (int k = 0; k < 32; ++k) {
|
|
210462
|
+
out[k] = static_cast<uint32_t>(in[k]);
|
|
210463
|
+
}
|
|
210464
|
+
}
|
|
210465
|
+
|
|
210466
|
+
void __fastpack33(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210467
|
+
Unroller<33>::Pack(in, out);
|
|
210468
|
+
}
|
|
210469
|
+
|
|
210470
|
+
void __fastpack34(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210471
|
+
Unroller<34>::Pack(in, out);
|
|
210472
|
+
}
|
|
210473
|
+
|
|
210474
|
+
void __fastpack35(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210475
|
+
Unroller<35>::Pack(in, out);
|
|
210476
|
+
}
|
|
210477
|
+
|
|
210478
|
+
void __fastpack36(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210479
|
+
Unroller<36>::Pack(in, out);
|
|
210480
|
+
}
|
|
210481
|
+
|
|
210482
|
+
void __fastpack37(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210483
|
+
Unroller<37>::Pack(in, out);
|
|
210484
|
+
}
|
|
210485
|
+
|
|
210486
|
+
void __fastpack38(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210487
|
+
Unroller<38>::Pack(in, out);
|
|
210488
|
+
}
|
|
210489
|
+
|
|
210490
|
+
void __fastpack39(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210491
|
+
Unroller<39>::Pack(in, out);
|
|
210492
|
+
}
|
|
210493
|
+
|
|
210494
|
+
void __fastpack40(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210495
|
+
Unroller<40>::Pack(in, out);
|
|
210496
|
+
}
|
|
210497
|
+
|
|
210498
|
+
void __fastpack41(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210499
|
+
Unroller<41>::Pack(in, out);
|
|
210500
|
+
}
|
|
210501
|
+
|
|
210502
|
+
void __fastpack42(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210503
|
+
Unroller<42>::Pack(in, out);
|
|
210504
|
+
}
|
|
210505
|
+
|
|
210506
|
+
void __fastpack43(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210507
|
+
Unroller<43>::Pack(in, out);
|
|
210508
|
+
}
|
|
210509
|
+
|
|
210510
|
+
void __fastpack44(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210511
|
+
Unroller<44>::Pack(in, out);
|
|
210512
|
+
}
|
|
210513
|
+
|
|
210514
|
+
void __fastpack45(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210515
|
+
Unroller<45>::Pack(in, out);
|
|
210516
|
+
}
|
|
210517
|
+
|
|
210518
|
+
void __fastpack46(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210519
|
+
Unroller<46>::Pack(in, out);
|
|
210520
|
+
}
|
|
210521
|
+
|
|
210522
|
+
void __fastpack47(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210523
|
+
Unroller<47>::Pack(in, out);
|
|
210524
|
+
}
|
|
210525
|
+
|
|
210526
|
+
void __fastpack48(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210527
|
+
Unroller<48>::Pack(in, out);
|
|
210528
|
+
}
|
|
210529
|
+
|
|
210530
|
+
void __fastpack49(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210531
|
+
Unroller<49>::Pack(in, out);
|
|
210532
|
+
}
|
|
210533
|
+
|
|
210534
|
+
void __fastpack50(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210535
|
+
Unroller<50>::Pack(in, out);
|
|
210536
|
+
}
|
|
210537
|
+
|
|
210538
|
+
void __fastpack51(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210539
|
+
Unroller<51>::Pack(in, out);
|
|
210540
|
+
}
|
|
210541
|
+
|
|
210542
|
+
void __fastpack52(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210543
|
+
Unroller<52>::Pack(in, out);
|
|
210544
|
+
}
|
|
210545
|
+
|
|
210546
|
+
void __fastpack53(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210547
|
+
Unroller<53>::Pack(in, out);
|
|
210548
|
+
}
|
|
210549
|
+
|
|
210550
|
+
void __fastpack54(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210551
|
+
Unroller<54>::Pack(in, out);
|
|
210552
|
+
}
|
|
210553
|
+
|
|
210554
|
+
void __fastpack55(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210555
|
+
Unroller<55>::Pack(in, out);
|
|
210556
|
+
}
|
|
210557
|
+
|
|
210558
|
+
void __fastpack56(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210559
|
+
Unroller<56>::Pack(in, out);
|
|
210560
|
+
}
|
|
210561
|
+
|
|
210562
|
+
void __fastpack57(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210563
|
+
Unroller<57>::Pack(in, out);
|
|
210564
|
+
}
|
|
210565
|
+
|
|
210566
|
+
void __fastpack58(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210567
|
+
Unroller<58>::Pack(in, out);
|
|
210568
|
+
}
|
|
210569
|
+
|
|
210570
|
+
void __fastpack59(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210571
|
+
Unroller<59>::Pack(in, out);
|
|
210572
|
+
}
|
|
210573
|
+
|
|
210574
|
+
void __fastpack60(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210575
|
+
Unroller<60>::Pack(in, out);
|
|
210576
|
+
}
|
|
210577
|
+
|
|
210578
|
+
void __fastpack61(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210579
|
+
Unroller<61>::Pack(in, out);
|
|
210580
|
+
}
|
|
210581
|
+
|
|
210582
|
+
void __fastpack62(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210583
|
+
Unroller<62>::Pack(in, out);
|
|
210584
|
+
}
|
|
210585
|
+
|
|
210586
|
+
void __fastpack63(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210587
|
+
Unroller<63>::Pack(in, out);
|
|
210588
|
+
}
|
|
210589
|
+
|
|
210590
|
+
void __fastpack64(const uint64_t *__restrict in, uint32_t *__restrict out) {
|
|
210591
|
+
for (int i = 0; i < 32; ++i) {
|
|
210592
|
+
out[2 * i] = static_cast<uint32_t>(in[i]);
|
|
210593
|
+
out[2 * i + 1] = in[i] >> 32;
|
|
210594
|
+
}
|
|
210595
|
+
}
|
|
210596
|
+
} // namespace internal
|
|
210597
|
+
} // namespace duckdb_fastpforlib
|
|
210598
|
+
|
|
210599
|
+
|
|
210600
|
+
// LICENSE_CHANGE_END
|
|
210601
|
+
|
|
210602
|
+
|
|
207504
210603
|
// LICENSE_CHANGE_BEGIN
|
|
207505
210604
|
// The following code up to LICENSE_CHANGE_END is subject to THIRD PARTY LICENSE #2
|
|
207506
210605
|
// See the end of this file for a list
|
|
@@ -257634,4 +260733,199 @@ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
|
|
257634
260733
|
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
|
257635
260734
|
POSSIBILITY OF SUCH DAMAGE.
|
|
257636
260735
|
|
|
260736
|
+
|
|
260737
|
+
### THIRD PARTY LICENSE #10 ###
|
|
260738
|
+
|
|
260739
|
+
Apache License
|
|
260740
|
+
Version 2.0, January 2004
|
|
260741
|
+
http://www.apache.org/licenses/
|
|
260742
|
+
|
|
260743
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
260744
|
+
|
|
260745
|
+
1. Definitions.
|
|
260746
|
+
|
|
260747
|
+
"License" shall mean the terms and conditions for use, reproduction, and
|
|
260748
|
+
distribution as defined by Sections 1 through 9 of this document.
|
|
260749
|
+
|
|
260750
|
+
"Licensor" shall mean the copyright owner or entity authorized by the copyright
|
|
260751
|
+
owner that is granting the License.
|
|
260752
|
+
|
|
260753
|
+
"Legal Entity" shall mean the union of the acting entity and all other entities
|
|
260754
|
+
that control, are controlled by, or are under common control with that entity.
|
|
260755
|
+
For the purposes of this definition, "control" means (i) the power, direct or
|
|
260756
|
+
indirect, to cause the direction or management of such entity, whether by
|
|
260757
|
+
contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
260758
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
260759
|
+
|
|
260760
|
+
"You" (or "Your") shall mean an individual or Legal Entity exercising
|
|
260761
|
+
permissions granted by this License.
|
|
260762
|
+
|
|
260763
|
+
"Source" form shall mean the preferred form for making modifications, including
|
|
260764
|
+
but not limited to software source code, documentation source, and configuration
|
|
260765
|
+
files.
|
|
260766
|
+
|
|
260767
|
+
"Object" form shall mean any form resulting from mechanical transformation or
|
|
260768
|
+
translation of a Source form, including but not limited to compiled object code,
|
|
260769
|
+
generated documentation, and conversions to other media types.
|
|
260770
|
+
|
|
260771
|
+
"Work" shall mean the work of authorship, whether in Source or Object form, made
|
|
260772
|
+
available under the License, as indicated by a copyright notice that is included
|
|
260773
|
+
in or attached to the work (an example is provided in the Appendix below).
|
|
260774
|
+
|
|
260775
|
+
"Derivative Works" shall mean any work, whether in Source or Object form, that
|
|
260776
|
+
is based on (or derived from) the Work and for which the editorial revisions,
|
|
260777
|
+
annotations, elaborations, or other modifications represent, as a whole, an
|
|
260778
|
+
original work of authorship. For the purposes of this License, Derivative Works
|
|
260779
|
+
shall not include works that remain separable from, or merely link (or bind by
|
|
260780
|
+
name) to the interfaces of, the Work and Derivative Works thereof.
|
|
260781
|
+
|
|
260782
|
+
"Contribution" shall mean any work of authorship, including the original version
|
|
260783
|
+
of the Work and any modifications or additions to that Work or Derivative Works
|
|
260784
|
+
thereof, that is intentionally submitted to Licensor for inclusion in the Work
|
|
260785
|
+
by the copyright owner or by an individual or Legal Entity authorized to submit
|
|
260786
|
+
on behalf of the copyright owner. For the purposes of this definition,
|
|
260787
|
+
"submitted" means any form of electronic, verbal, or written communication sent
|
|
260788
|
+
to the Licensor or its representatives, including but not limited to
|
|
260789
|
+
communication on electronic mailing lists, source code control systems, and
|
|
260790
|
+
issue tracking systems that are managed by, or on behalf of, the Licensor for
|
|
260791
|
+
the purpose of discussing and improving the Work, but excluding communication
|
|
260792
|
+
that is conspicuously marked or otherwise designated in writing by the copyright
|
|
260793
|
+
owner as "Not a Contribution."
|
|
260794
|
+
|
|
260795
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity on behalf
|
|
260796
|
+
of whom a Contribution has been received by Licensor and subsequently
|
|
260797
|
+
incorporated within the Work.
|
|
260798
|
+
|
|
260799
|
+
2. Grant of Copyright License.
|
|
260800
|
+
|
|
260801
|
+
Subject to the terms and conditions of this License, each Contributor hereby
|
|
260802
|
+
grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
|
|
260803
|
+
irrevocable copyright license to reproduce, prepare Derivative Works of,
|
|
260804
|
+
publicly display, publicly perform, sublicense, and distribute the Work and such
|
|
260805
|
+
Derivative Works in Source or Object form.
|
|
260806
|
+
|
|
260807
|
+
3. Grant of Patent License.
|
|
260808
|
+
|
|
260809
|
+
Subject to the terms and conditions of this License, each Contributor hereby
|
|
260810
|
+
grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
|
|
260811
|
+
irrevocable (except as stated in this section) patent license to make, have
|
|
260812
|
+
made, use, offer to sell, sell, import, and otherwise transfer the Work, where
|
|
260813
|
+
such license applies only to those patent claims licensable by such Contributor
|
|
260814
|
+
that are necessarily infringed by their Contribution(s) alone or by combination
|
|
260815
|
+
of their Contribution(s) with the Work to which such Contribution(s) was
|
|
260816
|
+
submitted. If You institute patent litigation against any entity (including a
|
|
260817
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work or a
|
|
260818
|
+
Contribution incorporated within the Work constitutes direct or contributory
|
|
260819
|
+
patent infringement, then any patent licenses granted to You under this License
|
|
260820
|
+
for that Work shall terminate as of the date such litigation is filed.
|
|
260821
|
+
|
|
260822
|
+
4. Redistribution.
|
|
260823
|
+
|
|
260824
|
+
You may reproduce and distribute copies of the Work or Derivative Works thereof
|
|
260825
|
+
in any medium, with or without modifications, and in Source or Object form,
|
|
260826
|
+
provided that You meet the following conditions:
|
|
260827
|
+
|
|
260828
|
+
You must give any other recipients of the Work or Derivative Works a copy of
|
|
260829
|
+
this License; and
|
|
260830
|
+
You must cause any modified files to carry prominent notices stating that You
|
|
260831
|
+
changed the files; and
|
|
260832
|
+
You must retain, in the Source form of any Derivative Works that You distribute,
|
|
260833
|
+
all copyright, patent, trademark, and attribution notices from the Source form
|
|
260834
|
+
of the Work, excluding those notices that do not pertain to any part of the
|
|
260835
|
+
Derivative Works; and
|
|
260836
|
+
If the Work includes a "NOTICE" text file as part of its distribution, then any
|
|
260837
|
+
Derivative Works that You distribute must include a readable copy of the
|
|
260838
|
+
attribution notices contained within such NOTICE file, excluding those notices
|
|
260839
|
+
that do not pertain to any part of the Derivative Works, in at least one of the
|
|
260840
|
+
following places: within a NOTICE text file distributed as part of the
|
|
260841
|
+
Derivative Works; within the Source form or documentation, if provided along
|
|
260842
|
+
with the Derivative Works; or, within a display generated by the Derivative
|
|
260843
|
+
Works, if and wherever such third-party notices normally appear. The contents of
|
|
260844
|
+
the NOTICE file are for informational purposes only and do not modify the
|
|
260845
|
+
License. You may add Your own attribution notices within Derivative Works that
|
|
260846
|
+
You distribute, alongside or as an addendum to the NOTICE text from the Work,
|
|
260847
|
+
provided that such additional attribution notices cannot be construed as
|
|
260848
|
+
modifying the License.
|
|
260849
|
+
You may add Your own copyright statement to Your modifications and may provide
|
|
260850
|
+
additional or different license terms and conditions for use, reproduction, or
|
|
260851
|
+
distribution of Your modifications, or for any such Derivative Works as a whole,
|
|
260852
|
+
provided Your use, reproduction, and distribution of the Work otherwise complies
|
|
260853
|
+
with the conditions stated in this License.
|
|
260854
|
+
|
|
260855
|
+
5. Submission of Contributions.
|
|
260856
|
+
|
|
260857
|
+
Unless You explicitly state otherwise, any Contribution intentionally submitted
|
|
260858
|
+
for inclusion in the Work by You to the Licensor shall be under the terms and
|
|
260859
|
+
conditions of this License, without any additional terms or conditions.
|
|
260860
|
+
Notwithstanding the above, nothing herein shall supersede or modify the terms of
|
|
260861
|
+
any separate license agreement you may have executed with Licensor regarding
|
|
260862
|
+
such Contributions.
|
|
260863
|
+
|
|
260864
|
+
6. Trademarks.
|
|
260865
|
+
|
|
260866
|
+
This License does not grant permission to use the trade names, trademarks,
|
|
260867
|
+
service marks, or product names of the Licensor, except as required for
|
|
260868
|
+
reasonable and customary use in describing the origin of the Work and
|
|
260869
|
+
reproducing the content of the NOTICE file.
|
|
260870
|
+
|
|
260871
|
+
7. Disclaimer of Warranty.
|
|
260872
|
+
|
|
260873
|
+
Unless required by applicable law or agreed to in writing, Licensor provides the
|
|
260874
|
+
Work (and each Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
260875
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied,
|
|
260876
|
+
including, without limitation, any warranties or conditions of TITLE,
|
|
260877
|
+
NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are
|
|
260878
|
+
solely responsible for determining the appropriateness of using or
|
|
260879
|
+
redistributing the Work and assume any risks associated with Your exercise of
|
|
260880
|
+
permissions under this License.
|
|
260881
|
+
|
|
260882
|
+
8. Limitation of Liability.
|
|
260883
|
+
|
|
260884
|
+
In no event and under no legal theory, whether in tort (including negligence),
|
|
260885
|
+
contract, or otherwise, unless required by applicable law (such as deliberate
|
|
260886
|
+
and grossly negligent acts) or agreed to in writing, shall any Contributor be
|
|
260887
|
+
liable to You for damages, including any direct, indirect, special, incidental,
|
|
260888
|
+
or consequential damages of any character arising as a result of this License or
|
|
260889
|
+
out of the use or inability to use the Work (including but not limited to
|
|
260890
|
+
damages for loss of goodwill, work stoppage, computer failure or malfunction, or
|
|
260891
|
+
any and all other commercial damages or losses), even if such Contributor has
|
|
260892
|
+
been advised of the possibility of such damages.
|
|
260893
|
+
|
|
260894
|
+
9. Accepting Warranty or Additional Liability.
|
|
260895
|
+
|
|
260896
|
+
While redistributing the Work or Derivative Works thereof, You may choose to
|
|
260897
|
+
offer, and charge a fee for, acceptance of support, warranty, indemnity, or
|
|
260898
|
+
other liability obligations and/or rights consistent with this License. However,
|
|
260899
|
+
in accepting such obligations, You may act only on Your own behalf and on Your
|
|
260900
|
+
sole responsibility, not on behalf of any other Contributor, and only if You
|
|
260901
|
+
agree to indemnify, defend, and hold each Contributor harmless for any liability
|
|
260902
|
+
incurred by, or claims asserted against, such Contributor by reason of your
|
|
260903
|
+
accepting any such warranty or additional liability.
|
|
260904
|
+
|
|
260905
|
+
END OF TERMS AND CONDITIONS
|
|
260906
|
+
|
|
260907
|
+
APPENDIX: How to apply the Apache License to your work
|
|
260908
|
+
|
|
260909
|
+
To apply the Apache License to your work, attach the following boilerplate
|
|
260910
|
+
notice, with the fields enclosed by brackets "[]" replaced with your own
|
|
260911
|
+
identifying information. (Don't include the brackets!) The text should be
|
|
260912
|
+
enclosed in the appropriate comment syntax for the file format. We also
|
|
260913
|
+
recommend that a file or class name and description of purpose be included on
|
|
260914
|
+
the same "printed page" as the copyright notice for easier identification within
|
|
260915
|
+
third-party archives.
|
|
260916
|
+
|
|
260917
|
+
Copyright [yyyy] [name of copyright owner]
|
|
260918
|
+
|
|
260919
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
260920
|
+
you may not use this file except in compliance with the License.
|
|
260921
|
+
You may obtain a copy of the License at
|
|
260922
|
+
|
|
260923
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
260924
|
+
|
|
260925
|
+
Unless required by applicable law or agreed to in writing, software
|
|
260926
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
260927
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
260928
|
+
See the License for the specific language governing permissions and
|
|
260929
|
+
limitations under the License.
|
|
260930
|
+
|
|
257637
260931
|
*/
|