zstdlib 0.7.0-x64-mingw32 → 0.8.0-x64-mingw32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGES.md +5 -0
- data/ext/zstdlib/extconf.rb +1 -1
- data/ext/zstdlib/ruby/zlib-3.0/zstdlib.c +4994 -0
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/bitstream.h +25 -16
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/compiler.h +118 -4
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/cpu.h +1 -3
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/debug.c +1 -1
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/debug.h +12 -19
- data/ext/zstdlib/zstd-1.5.0/lib/common/entropy_common.c +362 -0
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/error_private.c +2 -1
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/error_private.h +3 -3
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/fse.h +40 -12
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/fse_decompress.c +139 -22
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/huf.h +29 -7
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/mem.h +69 -98
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/pool.c +23 -17
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/pool.h +2 -2
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/threading.c +6 -5
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/threading.h +0 -0
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/xxhash.c +20 -60
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/xxhash.h +2 -2
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/zstd_common.c +10 -10
- data/ext/zstdlib/zstd-1.5.0/lib/common/zstd_deps.h +111 -0
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/common/zstd_internal.h +105 -62
- data/ext/zstdlib/zstd-1.5.0/lib/common/zstd_trace.h +154 -0
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/fse_compress.c +31 -24
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/hist.c +27 -29
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/hist.h +2 -2
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/huf_compress.c +265 -126
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_compress.c +2843 -728
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_compress_internal.h +305 -63
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_compress_literals.c +8 -8
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_compress_literals.h +1 -1
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_compress_sequences.c +29 -7
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_compress_sequences.h +1 -1
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_compress_superblock.c +22 -295
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_compress_superblock.h +1 -1
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_cwksp.h +204 -67
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_double_fast.c +25 -25
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_double_fast.h +1 -1
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_fast.c +23 -23
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_fast.h +1 -1
- data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_lazy.c +2184 -0
- data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_lazy.h +125 -0
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_ldm.c +314 -211
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_ldm.h +9 -2
- data/ext/zstdlib/zstd-1.5.0/lib/compress/zstd_ldm_geartab.h +103 -0
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_opt.c +191 -46
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstd_opt.h +1 -1
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/compress/zstdmt_compress.c +93 -415
- data/ext/zstdlib/zstd-1.5.0/lib/compress/zstdmt_compress.h +110 -0
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/decompress/huf_decompress.c +342 -239
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/decompress/zstd_ddict.c +9 -9
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/decompress/zstd_ddict.h +2 -2
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/decompress/zstd_decompress.c +369 -87
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/decompress/zstd_decompress_block.c +191 -75
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/decompress/zstd_decompress_block.h +6 -3
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/decompress/zstd_decompress_internal.h +27 -11
- data/ext/zstdlib/zstd-1.5.0/lib/zdict.h +452 -0
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/lib/zstd.h +568 -126
- data/ext/zstdlib/{zstd-1.4.5/lib/common → zstd-1.5.0/lib}/zstd_errors.h +2 -1
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/zlibWrapper/gzclose.c +0 -0
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/zlibWrapper/gzcompatibility.h +1 -1
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/zlibWrapper/gzguts.h +0 -0
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/zlibWrapper/gzlib.c +0 -0
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/zlibWrapper/gzread.c +0 -0
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/zlibWrapper/gzwrite.c +0 -0
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/zlibWrapper/zstd_zlibwrapper.c +126 -44
- data/ext/zstdlib/{zstd-1.4.5 → zstd-1.5.0}/zlibWrapper/zstd_zlibwrapper.h +1 -1
- data/lib/2.2/zstdlib.so +0 -0
- data/lib/2.3/zstdlib.so +0 -0
- data/lib/2.4/zstdlib.so +0 -0
- data/lib/2.5/zstdlib.so +0 -0
- data/lib/2.6/zstdlib.so +0 -0
- data/lib/2.7/zstdlib.so +0 -0
- metadata +69 -64
- data/ext/zstdlib/zstd-1.4.5/lib/common/entropy_common.c +0 -216
- data/ext/zstdlib/zstd-1.4.5/lib/compress/zstd_lazy.c +0 -1138
- data/ext/zstdlib/zstd-1.4.5/lib/compress/zstd_lazy.h +0 -67
- data/ext/zstdlib/zstd-1.4.5/lib/compress/zstdmt_compress.h +0 -192
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c)
|
2
|
+
* Copyright (c) Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -66,6 +66,7 @@ size_t ZSTD_ldm_generateSequences(
|
|
66
66
|
*/
|
67
67
|
size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
|
68
68
|
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
69
|
+
ZSTD_useRowMatchFinderMode_e useRowMatchFinder,
|
69
70
|
void const* src, size_t srcSize);
|
70
71
|
|
71
72
|
/**
|
@@ -73,11 +74,17 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
|
|
73
74
|
*
|
74
75
|
* Skip past `srcSize` bytes worth of sequences in `rawSeqStore`.
|
75
76
|
* Avoids emitting matches less than `minMatch` bytes.
|
76
|
-
* Must be called for data
|
77
|
+
* Must be called for data that is not passed to ZSTD_ldm_blockCompress().
|
77
78
|
*/
|
78
79
|
void ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize,
|
79
80
|
U32 const minMatch);
|
80
81
|
|
82
|
+
/* ZSTD_ldm_skipRawSeqStoreBytes():
|
83
|
+
* Moves forward in rawSeqStore by nbBytes, updating fields 'pos' and 'posInSequence'.
|
84
|
+
* Not to be used in conjunction with ZSTD_ldm_skipSequences().
|
85
|
+
* Must be called for data with is not passed to ZSTD_ldm_blockCompress().
|
86
|
+
*/
|
87
|
+
void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes);
|
81
88
|
|
82
89
|
/** ZSTD_ldm_getTableSize() :
|
83
90
|
* Estimate the space needed for long distance matching tables or 0 if LDM is
|
@@ -0,0 +1,103 @@
|
|
1
|
+
/*
|
2
|
+
* Copyright (c) Yann Collet, Facebook, Inc.
|
3
|
+
* All rights reserved.
|
4
|
+
*
|
5
|
+
* This source code is licensed under both the BSD-style license (found in the
|
6
|
+
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
7
|
+
* in the COPYING file in the root directory of this source tree).
|
8
|
+
* You may select, at your option, one of the above-listed licenses.
|
9
|
+
*/
|
10
|
+
|
11
|
+
#ifndef ZSTD_LDM_GEARTAB_H
|
12
|
+
#define ZSTD_LDM_GEARTAB_H
|
13
|
+
|
14
|
+
static U64 ZSTD_ldm_gearTab[256] = {
|
15
|
+
0xf5b8f72c5f77775c, 0x84935f266b7ac412, 0xb647ada9ca730ccc,
|
16
|
+
0xb065bb4b114fb1de, 0x34584e7e8c3a9fd0, 0x4e97e17c6ae26b05,
|
17
|
+
0x3a03d743bc99a604, 0xcecd042422c4044f, 0x76de76c58524259e,
|
18
|
+
0x9c8528f65badeaca, 0x86563706e2097529, 0x2902475fa375d889,
|
19
|
+
0xafb32a9739a5ebe6, 0xce2714da3883e639, 0x21eaf821722e69e,
|
20
|
+
0x37b628620b628, 0x49a8d455d88caf5, 0x8556d711e6958140,
|
21
|
+
0x4f7ae74fc605c1f, 0x829f0c3468bd3a20, 0x4ffdc885c625179e,
|
22
|
+
0x8473de048a3daf1b, 0x51008822b05646b2, 0x69d75d12b2d1cc5f,
|
23
|
+
0x8c9d4a19159154bc, 0xc3cc10f4abbd4003, 0xd06ddc1cecb97391,
|
24
|
+
0xbe48e6e7ed80302e, 0x3481db31cee03547, 0xacc3f67cdaa1d210,
|
25
|
+
0x65cb771d8c7f96cc, 0x8eb27177055723dd, 0xc789950d44cd94be,
|
26
|
+
0x934feadc3700b12b, 0x5e485f11edbdf182, 0x1e2e2a46fd64767a,
|
27
|
+
0x2969ca71d82efa7c, 0x9d46e9935ebbba2e, 0xe056b67e05e6822b,
|
28
|
+
0x94d73f55739d03a0, 0xcd7010bdb69b5a03, 0x455ef9fcd79b82f4,
|
29
|
+
0x869cb54a8749c161, 0x38d1a4fa6185d225, 0xb475166f94bbe9bb,
|
30
|
+
0xa4143548720959f1, 0x7aed4780ba6b26ba, 0xd0ce264439e02312,
|
31
|
+
0x84366d746078d508, 0xa8ce973c72ed17be, 0x21c323a29a430b01,
|
32
|
+
0x9962d617e3af80ee, 0xab0ce91d9c8cf75b, 0x530e8ee6d19a4dbc,
|
33
|
+
0x2ef68c0cf53f5d72, 0xc03a681640a85506, 0x496e4e9f9c310967,
|
34
|
+
0x78580472b59b14a0, 0x273824c23b388577, 0x66bf923ad45cb553,
|
35
|
+
0x47ae1a5a2492ba86, 0x35e304569e229659, 0x4765182a46870b6f,
|
36
|
+
0x6cbab625e9099412, 0xddac9a2e598522c1, 0x7172086e666624f2,
|
37
|
+
0xdf5003ca503b7837, 0x88c0c1db78563d09, 0x58d51865acfc289d,
|
38
|
+
0x177671aec65224f1, 0xfb79d8a241e967d7, 0x2be1e101cad9a49a,
|
39
|
+
0x6625682f6e29186b, 0x399553457ac06e50, 0x35dffb4c23abb74,
|
40
|
+
0x429db2591f54aade, 0xc52802a8037d1009, 0x6acb27381f0b25f3,
|
41
|
+
0xf45e2551ee4f823b, 0x8b0ea2d99580c2f7, 0x3bed519cbcb4e1e1,
|
42
|
+
0xff452823dbb010a, 0x9d42ed614f3dd267, 0x5b9313c06257c57b,
|
43
|
+
0xa114b8008b5e1442, 0xc1fe311c11c13d4b, 0x66e8763ea34c5568,
|
44
|
+
0x8b982af1c262f05d, 0xee8876faaa75fbb7, 0x8a62a4d0d172bb2a,
|
45
|
+
0xc13d94a3b7449a97, 0x6dbbba9dc15d037c, 0xc786101f1d92e0f1,
|
46
|
+
0xd78681a907a0b79b, 0xf61aaf2962c9abb9, 0x2cfd16fcd3cb7ad9,
|
47
|
+
0x868c5b6744624d21, 0x25e650899c74ddd7, 0xba042af4a7c37463,
|
48
|
+
0x4eb1a539465a3eca, 0xbe09dbf03b05d5ca, 0x774e5a362b5472ba,
|
49
|
+
0x47a1221229d183cd, 0x504b0ca18ef5a2df, 0xdffbdfbde2456eb9,
|
50
|
+
0x46cd2b2fbee34634, 0xf2aef8fe819d98c3, 0x357f5276d4599d61,
|
51
|
+
0x24a5483879c453e3, 0x88026889192b4b9, 0x28da96671782dbec,
|
52
|
+
0x4ef37c40588e9aaa, 0x8837b90651bc9fb3, 0xc164f741d3f0e5d6,
|
53
|
+
0xbc135a0a704b70ba, 0x69cd868f7622ada, 0xbc37ba89e0b9c0ab,
|
54
|
+
0x47c14a01323552f6, 0x4f00794bacee98bb, 0x7107de7d637a69d5,
|
55
|
+
0x88af793bb6f2255e, 0xf3c6466b8799b598, 0xc288c616aa7f3b59,
|
56
|
+
0x81ca63cf42fca3fd, 0x88d85ace36a2674b, 0xd056bd3792389e7,
|
57
|
+
0xe55c396c4e9dd32d, 0xbefb504571e6c0a6, 0x96ab32115e91e8cc,
|
58
|
+
0xbf8acb18de8f38d1, 0x66dae58801672606, 0x833b6017872317fb,
|
59
|
+
0xb87c16f2d1c92864, 0xdb766a74e58b669c, 0x89659f85c61417be,
|
60
|
+
0xc8daad856011ea0c, 0x76a4b565b6fe7eae, 0xa469d085f6237312,
|
61
|
+
0xaaf0365683a3e96c, 0x4dbb746f8424f7b8, 0x638755af4e4acc1,
|
62
|
+
0x3d7807f5bde64486, 0x17be6d8f5bbb7639, 0x903f0cd44dc35dc,
|
63
|
+
0x67b672eafdf1196c, 0xa676ff93ed4c82f1, 0x521d1004c5053d9d,
|
64
|
+
0x37ba9ad09ccc9202, 0x84e54d297aacfb51, 0xa0b4b776a143445,
|
65
|
+
0x820d471e20b348e, 0x1874383cb83d46dc, 0x97edeec7a1efe11c,
|
66
|
+
0xb330e50b1bdc42aa, 0x1dd91955ce70e032, 0xa514cdb88f2939d5,
|
67
|
+
0x2791233fd90db9d3, 0x7b670a4cc50f7a9b, 0x77c07d2a05c6dfa5,
|
68
|
+
0xe3778b6646d0a6fa, 0xb39c8eda47b56749, 0x933ed448addbef28,
|
69
|
+
0xaf846af6ab7d0bf4, 0xe5af208eb666e49, 0x5e6622f73534cd6a,
|
70
|
+
0x297daeca42ef5b6e, 0x862daef3d35539a6, 0xe68722498f8e1ea9,
|
71
|
+
0x981c53093dc0d572, 0xfa09b0bfbf86fbf5, 0x30b1e96166219f15,
|
72
|
+
0x70e7d466bdc4fb83, 0x5a66736e35f2a8e9, 0xcddb59d2b7c1baef,
|
73
|
+
0xd6c7d247d26d8996, 0xea4e39eac8de1ba3, 0x539c8bb19fa3aff2,
|
74
|
+
0x9f90e4c5fd508d8, 0xa34e5956fbaf3385, 0x2e2f8e151d3ef375,
|
75
|
+
0x173691e9b83faec1, 0xb85a8d56bf016379, 0x8382381267408ae3,
|
76
|
+
0xb90f901bbdc0096d, 0x7c6ad32933bcec65, 0x76bb5e2f2c8ad595,
|
77
|
+
0x390f851a6cf46d28, 0xc3e6064da1c2da72, 0xc52a0c101cfa5389,
|
78
|
+
0xd78eaf84a3fbc530, 0x3781b9e2288b997e, 0x73c2f6dea83d05c4,
|
79
|
+
0x4228e364c5b5ed7, 0x9d7a3edf0da43911, 0x8edcfeda24686756,
|
80
|
+
0x5e7667a7b7a9b3a1, 0x4c4f389fa143791d, 0xb08bc1023da7cddc,
|
81
|
+
0x7ab4be3ae529b1cc, 0x754e6132dbe74ff9, 0x71635442a839df45,
|
82
|
+
0x2f6fb1643fbe52de, 0x961e0a42cf7a8177, 0xf3b45d83d89ef2ea,
|
83
|
+
0xee3de4cf4a6e3e9b, 0xcd6848542c3295e7, 0xe4cee1664c78662f,
|
84
|
+
0x9947548b474c68c4, 0x25d73777a5ed8b0b, 0xc915b1d636b7fc,
|
85
|
+
0x21c2ba75d9b0d2da, 0x5f6b5dcf608a64a1, 0xdcf333255ff9570c,
|
86
|
+
0x633b922418ced4ee, 0xc136dde0b004b34a, 0x58cc83b05d4b2f5a,
|
87
|
+
0x5eb424dda28e42d2, 0x62df47369739cd98, 0xb4e0b42485e4ce17,
|
88
|
+
0x16e1f0c1f9a8d1e7, 0x8ec3916707560ebf, 0x62ba6e2df2cc9db3,
|
89
|
+
0xcbf9f4ff77d83a16, 0x78d9d7d07d2bbcc4, 0xef554ce1e02c41f4,
|
90
|
+
0x8d7581127eccf94d, 0xa9b53336cb3c8a05, 0x38c42c0bf45c4f91,
|
91
|
+
0x640893cdf4488863, 0x80ec34bc575ea568, 0x39f324f5b48eaa40,
|
92
|
+
0xe9d9ed1f8eff527f, 0x9224fc058cc5a214, 0xbaba00b04cfe7741,
|
93
|
+
0x309a9f120fcf52af, 0xa558f3ec65626212, 0x424bec8b7adabe2f,
|
94
|
+
0x41622513a6aea433, 0xb88da2d5324ca798, 0xd287733b245528a4,
|
95
|
+
0x9a44697e6d68aec3, 0x7b1093be2f49bb28, 0x50bbec632e3d8aad,
|
96
|
+
0x6cd90723e1ea8283, 0x897b9e7431b02bf3, 0x219efdcb338a7047,
|
97
|
+
0x3b0311f0a27c0656, 0xdb17bf91c0db96e7, 0x8cd4fd6b4e85a5b2,
|
98
|
+
0xfab071054ba6409d, 0x40d6fe831fa9dfd9, 0xaf358debad7d791e,
|
99
|
+
0xeb8d0e25a65e3e58, 0xbbcbd3df14e08580, 0xcf751f27ecdab2b,
|
100
|
+
0x2b4da14f2613d8f4
|
101
|
+
};
|
102
|
+
|
103
|
+
#endif /* ZSTD_LDM_GEARTAB_H */
|
@@ -1,5 +1,5 @@
|
|
1
1
|
/*
|
2
|
-
* Copyright (c)
|
2
|
+
* Copyright (c) Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
|
3
3
|
* All rights reserved.
|
4
4
|
*
|
5
5
|
* This source code is licensed under both the BSD-style license (found in the
|
@@ -386,32 +386,32 @@ static U32 ZSTD_insertBt1(
|
|
386
386
|
const BYTE* const dictEnd = dictBase + dictLimit;
|
387
387
|
const BYTE* const prefixStart = base + dictLimit;
|
388
388
|
const BYTE* match;
|
389
|
-
const U32
|
390
|
-
const U32 btLow = btMask >=
|
391
|
-
U32* smallerPtr = bt + 2*(
|
389
|
+
const U32 curr = (U32)(ip-base);
|
390
|
+
const U32 btLow = btMask >= curr ? 0 : curr - btMask;
|
391
|
+
U32* smallerPtr = bt + 2*(curr&btMask);
|
392
392
|
U32* largerPtr = smallerPtr + 1;
|
393
393
|
U32 dummy32; /* to be nullified at the end */
|
394
394
|
U32 const windowLow = ms->window.lowLimit;
|
395
|
-
U32 matchEndIdx =
|
395
|
+
U32 matchEndIdx = curr+8+1;
|
396
396
|
size_t bestLength = 8;
|
397
397
|
U32 nbCompares = 1U << cParams->searchLog;
|
398
398
|
#ifdef ZSTD_C_PREDICT
|
399
|
-
U32 predictedSmall = *(bt + 2*((
|
400
|
-
U32 predictedLarge = *(bt + 2*((
|
399
|
+
U32 predictedSmall = *(bt + 2*((curr-1)&btMask) + 0);
|
400
|
+
U32 predictedLarge = *(bt + 2*((curr-1)&btMask) + 1);
|
401
401
|
predictedSmall += (predictedSmall>0);
|
402
402
|
predictedLarge += (predictedLarge>0);
|
403
403
|
#endif /* ZSTD_C_PREDICT */
|
404
404
|
|
405
|
-
DEBUGLOG(8, "ZSTD_insertBt1 (%u)",
|
405
|
+
DEBUGLOG(8, "ZSTD_insertBt1 (%u)", curr);
|
406
406
|
|
407
407
|
assert(ip <= iend-8); /* required for h calculation */
|
408
|
-
hashTable[h] =
|
408
|
+
hashTable[h] = curr; /* Update Hash Table */
|
409
409
|
|
410
410
|
assert(windowLow > 0);
|
411
411
|
while (nbCompares-- && (matchIndex >= windowLow)) {
|
412
412
|
U32* const nextPtr = bt + 2*(matchIndex & btMask);
|
413
413
|
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
|
414
|
-
assert(matchIndex <
|
414
|
+
assert(matchIndex < curr);
|
415
415
|
|
416
416
|
#ifdef ZSTD_C_PREDICT /* note : can create issues when hlog small <= 11 */
|
417
417
|
const U32* predictPtr = bt + 2*((matchIndex-1) & btMask); /* written this way, as bt is a roll buffer */
|
@@ -474,8 +474,8 @@ static U32 ZSTD_insertBt1(
|
|
474
474
|
*smallerPtr = *largerPtr = 0;
|
475
475
|
{ U32 positions = 0;
|
476
476
|
if (bestLength > 384) positions = MIN(192, (U32)(bestLength - 384)); /* speed optimization */
|
477
|
-
assert(matchEndIdx >
|
478
|
-
return MAX(positions, matchEndIdx - (
|
477
|
+
assert(matchEndIdx > curr + 8);
|
478
|
+
return MAX(positions, matchEndIdx - (curr + 8));
|
479
479
|
}
|
480
480
|
}
|
481
481
|
|
@@ -519,7 +519,7 @@ U32 ZSTD_insertBtAndGetAllMatches (
|
|
519
519
|
const ZSTD_compressionParameters* const cParams = &ms->cParams;
|
520
520
|
U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1);
|
521
521
|
const BYTE* const base = ms->window.base;
|
522
|
-
U32 const
|
522
|
+
U32 const curr = (U32)(ip-base);
|
523
523
|
U32 const hashLog = cParams->hashLog;
|
524
524
|
U32 const minMatch = (mls==3) ? 3 : 4;
|
525
525
|
U32* const hashTable = ms->hashTable;
|
@@ -533,12 +533,12 @@ U32 ZSTD_insertBtAndGetAllMatches (
|
|
533
533
|
U32 const dictLimit = ms->window.dictLimit;
|
534
534
|
const BYTE* const dictEnd = dictBase + dictLimit;
|
535
535
|
const BYTE* const prefixStart = base + dictLimit;
|
536
|
-
U32 const btLow = (btMask >=
|
537
|
-
U32 const windowLow = ZSTD_getLowestMatchIndex(ms,
|
536
|
+
U32 const btLow = (btMask >= curr) ? 0 : curr - btMask;
|
537
|
+
U32 const windowLow = ZSTD_getLowestMatchIndex(ms, curr, cParams->windowLog);
|
538
538
|
U32 const matchLow = windowLow ? windowLow : 1;
|
539
|
-
U32* smallerPtr = bt + 2*(
|
540
|
-
U32* largerPtr = bt + 2*(
|
541
|
-
U32 matchEndIdx =
|
539
|
+
U32* smallerPtr = bt + 2*(curr&btMask);
|
540
|
+
U32* largerPtr = bt + 2*(curr&btMask) + 1;
|
541
|
+
U32 matchEndIdx = curr+8+1; /* farthest referenced position of any match => detects repetitive patterns */
|
542
542
|
U32 dummy32; /* to be nullified at the end */
|
543
543
|
U32 mnum = 0;
|
544
544
|
U32 nbCompares = 1U << cParams->searchLog;
|
@@ -557,7 +557,7 @@ U32 ZSTD_insertBtAndGetAllMatches (
|
|
557
557
|
U32 const dmsBtLow = dictMode == ZSTD_dictMatchState && dmsBtMask < dmsHighLimit - dmsLowLimit ? dmsHighLimit - dmsBtMask : dmsLowLimit;
|
558
558
|
|
559
559
|
size_t bestLength = lengthToBeat-1;
|
560
|
-
DEBUGLOG(8, "ZSTD_insertBtAndGetAllMatches: current=%u",
|
560
|
+
DEBUGLOG(8, "ZSTD_insertBtAndGetAllMatches: current=%u", curr);
|
561
561
|
|
562
562
|
/* check repCode */
|
563
563
|
assert(ll0 <= 1); /* necessarily 1 or 0 */
|
@@ -565,29 +565,29 @@ U32 ZSTD_insertBtAndGetAllMatches (
|
|
565
565
|
U32 repCode;
|
566
566
|
for (repCode = ll0; repCode < lastR; repCode++) {
|
567
567
|
U32 const repOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode];
|
568
|
-
U32 const repIndex =
|
568
|
+
U32 const repIndex = curr - repOffset;
|
569
569
|
U32 repLen = 0;
|
570
|
-
assert(
|
571
|
-
if (repOffset-1 /* intentional overflow, discards 0 and -1 */ <
|
570
|
+
assert(curr >= dictLimit);
|
571
|
+
if (repOffset-1 /* intentional overflow, discards 0 and -1 */ < curr-dictLimit) { /* equivalent to `curr > repIndex >= dictLimit` */
|
572
572
|
/* We must validate the repcode offset because when we're using a dictionary the
|
573
573
|
* valid offset range shrinks when the dictionary goes out of bounds.
|
574
574
|
*/
|
575
575
|
if ((repIndex >= windowLow) & (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(ip - repOffset, minMatch))) {
|
576
576
|
repLen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-repOffset, iLimit) + minMatch;
|
577
577
|
}
|
578
|
-
} else { /* repIndex < dictLimit || repIndex >=
|
578
|
+
} else { /* repIndex < dictLimit || repIndex >= curr */
|
579
579
|
const BYTE* const repMatch = dictMode == ZSTD_dictMatchState ?
|
580
580
|
dmsBase + repIndex - dmsIndexDelta :
|
581
581
|
dictBase + repIndex;
|
582
|
-
assert(
|
582
|
+
assert(curr >= windowLow);
|
583
583
|
if ( dictMode == ZSTD_extDict
|
584
|
-
&& ( ((repOffset-1) /*intentional overflow*/ <
|
584
|
+
&& ( ((repOffset-1) /*intentional overflow*/ < curr - windowLow) /* equivalent to `curr > repIndex >= windowLow` */
|
585
585
|
& (((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */)
|
586
586
|
&& (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) {
|
587
587
|
repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dictEnd, prefixStart) + minMatch;
|
588
588
|
}
|
589
589
|
if (dictMode == ZSTD_dictMatchState
|
590
|
-
&& ( ((repOffset-1) /*intentional overflow*/ <
|
590
|
+
&& ( ((repOffset-1) /*intentional overflow*/ < curr - (dmsLowLimit + dmsIndexDelta)) /* equivalent to `curr > repIndex >= dmsLowLimit` */
|
591
591
|
& ((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */
|
592
592
|
&& (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) {
|
593
593
|
repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dmsEnd, prefixStart) + minMatch;
|
@@ -609,7 +609,7 @@ U32 ZSTD_insertBtAndGetAllMatches (
|
|
609
609
|
if ((mls == 3) /*static*/ && (bestLength < mls)) {
|
610
610
|
U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3(ms, nextToUpdate3, ip);
|
611
611
|
if ((matchIndex3 >= matchLow)
|
612
|
-
& (
|
612
|
+
& (curr - matchIndex3 < (1<<18)) /*heuristic : longer distance likely too expensive*/ ) {
|
613
613
|
size_t mlen;
|
614
614
|
if ((dictMode == ZSTD_noDict) /*static*/ || (dictMode == ZSTD_dictMatchState) /*static*/ || (matchIndex3 >= dictLimit)) {
|
615
615
|
const BYTE* const match = base + matchIndex3;
|
@@ -624,26 +624,26 @@ U32 ZSTD_insertBtAndGetAllMatches (
|
|
624
624
|
DEBUGLOG(8, "found small match with hlog3, of length %u",
|
625
625
|
(U32)mlen);
|
626
626
|
bestLength = mlen;
|
627
|
-
assert(
|
627
|
+
assert(curr > matchIndex3);
|
628
628
|
assert(mnum==0); /* no prior solution */
|
629
|
-
matches[0].off = (
|
629
|
+
matches[0].off = (curr - matchIndex3) + ZSTD_REP_MOVE;
|
630
630
|
matches[0].len = (U32)mlen;
|
631
631
|
mnum = 1;
|
632
632
|
if ( (mlen > sufficient_len) |
|
633
633
|
(ip+mlen == iLimit) ) { /* best possible length */
|
634
|
-
ms->nextToUpdate =
|
634
|
+
ms->nextToUpdate = curr+1; /* skip insertion */
|
635
635
|
return 1;
|
636
636
|
} } }
|
637
637
|
/* no dictMatchState lookup: dicts don't have a populated HC3 table */
|
638
638
|
}
|
639
639
|
|
640
|
-
hashTable[h] =
|
640
|
+
hashTable[h] = curr; /* Update Hash Table */
|
641
641
|
|
642
642
|
while (nbCompares-- && (matchIndex >= matchLow)) {
|
643
643
|
U32* const nextPtr = bt + 2*(matchIndex & btMask);
|
644
644
|
const BYTE* match;
|
645
645
|
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
|
646
|
-
assert(
|
646
|
+
assert(curr > matchIndex);
|
647
647
|
|
648
648
|
if ((dictMode == ZSTD_noDict) || (dictMode == ZSTD_dictMatchState) || (matchIndex+matchLength >= dictLimit)) {
|
649
649
|
assert(matchIndex+matchLength >= dictLimit); /* ensure the condition is correct when !extDict */
|
@@ -660,12 +660,12 @@ U32 ZSTD_insertBtAndGetAllMatches (
|
|
660
660
|
|
661
661
|
if (matchLength > bestLength) {
|
662
662
|
DEBUGLOG(8, "found match of length %u at distance %u (offCode=%u)",
|
663
|
-
(U32)matchLength,
|
663
|
+
(U32)matchLength, curr - matchIndex, curr - matchIndex + ZSTD_REP_MOVE);
|
664
664
|
assert(matchEndIdx > matchIndex);
|
665
665
|
if (matchLength > matchEndIdx - matchIndex)
|
666
666
|
matchEndIdx = matchIndex + (U32)matchLength;
|
667
667
|
bestLength = matchLength;
|
668
|
-
matches[mnum].off = (
|
668
|
+
matches[mnum].off = (curr - matchIndex) + ZSTD_REP_MOVE;
|
669
669
|
matches[mnum].len = (U32)matchLength;
|
670
670
|
mnum++;
|
671
671
|
if ( (matchLength > ZSTD_OPT_NUM)
|
@@ -708,11 +708,11 @@ U32 ZSTD_insertBtAndGetAllMatches (
|
|
708
708
|
if (matchLength > bestLength) {
|
709
709
|
matchIndex = dictMatchIndex + dmsIndexDelta;
|
710
710
|
DEBUGLOG(8, "found dms match of length %u at distance %u (offCode=%u)",
|
711
|
-
(U32)matchLength,
|
711
|
+
(U32)matchLength, curr - matchIndex, curr - matchIndex + ZSTD_REP_MOVE);
|
712
712
|
if (matchLength > matchEndIdx - matchIndex)
|
713
713
|
matchEndIdx = matchIndex + (U32)matchLength;
|
714
714
|
bestLength = matchLength;
|
715
|
-
matches[mnum].off = (
|
715
|
+
matches[mnum].off = (curr - matchIndex) + ZSTD_REP_MOVE;
|
716
716
|
matches[mnum].len = (U32)matchLength;
|
717
717
|
mnum++;
|
718
718
|
if ( (matchLength > ZSTD_OPT_NUM)
|
@@ -733,7 +733,7 @@ U32 ZSTD_insertBtAndGetAllMatches (
|
|
733
733
|
}
|
734
734
|
}
|
735
735
|
|
736
|
-
assert(matchEndIdx >
|
736
|
+
assert(matchEndIdx > curr+8);
|
737
737
|
ms->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */
|
738
738
|
return mnum;
|
739
739
|
}
|
@@ -764,6 +764,140 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches (
|
|
764
764
|
}
|
765
765
|
}
|
766
766
|
|
767
|
+
/*************************
|
768
|
+
* LDM helper functions *
|
769
|
+
*************************/
|
770
|
+
|
771
|
+
/* Struct containing info needed to make decision about ldm inclusion */
|
772
|
+
typedef struct {
|
773
|
+
rawSeqStore_t seqStore; /* External match candidates store for this block */
|
774
|
+
U32 startPosInBlock; /* Start position of the current match candidate */
|
775
|
+
U32 endPosInBlock; /* End position of the current match candidate */
|
776
|
+
U32 offset; /* Offset of the match candidate */
|
777
|
+
} ZSTD_optLdm_t;
|
778
|
+
|
779
|
+
/* ZSTD_optLdm_skipRawSeqStoreBytes():
|
780
|
+
* Moves forward in rawSeqStore by nbBytes, which will update the fields 'pos' and 'posInSequence'.
|
781
|
+
*/
|
782
|
+
static void ZSTD_optLdm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) {
|
783
|
+
U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes);
|
784
|
+
while (currPos && rawSeqStore->pos < rawSeqStore->size) {
|
785
|
+
rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos];
|
786
|
+
if (currPos >= currSeq.litLength + currSeq.matchLength) {
|
787
|
+
currPos -= currSeq.litLength + currSeq.matchLength;
|
788
|
+
rawSeqStore->pos++;
|
789
|
+
} else {
|
790
|
+
rawSeqStore->posInSequence = currPos;
|
791
|
+
break;
|
792
|
+
}
|
793
|
+
}
|
794
|
+
if (currPos == 0 || rawSeqStore->pos == rawSeqStore->size) {
|
795
|
+
rawSeqStore->posInSequence = 0;
|
796
|
+
}
|
797
|
+
}
|
798
|
+
|
799
|
+
/* ZSTD_opt_getNextMatchAndUpdateSeqStore():
|
800
|
+
* Calculates the beginning and end of the next match in the current block.
|
801
|
+
* Updates 'pos' and 'posInSequence' of the ldmSeqStore.
|
802
|
+
*/
|
803
|
+
static void ZSTD_opt_getNextMatchAndUpdateSeqStore(ZSTD_optLdm_t* optLdm, U32 currPosInBlock,
|
804
|
+
U32 blockBytesRemaining) {
|
805
|
+
rawSeq currSeq;
|
806
|
+
U32 currBlockEndPos;
|
807
|
+
U32 literalsBytesRemaining;
|
808
|
+
U32 matchBytesRemaining;
|
809
|
+
|
810
|
+
/* Setting match end position to MAX to ensure we never use an LDM during this block */
|
811
|
+
if (optLdm->seqStore.size == 0 || optLdm->seqStore.pos >= optLdm->seqStore.size) {
|
812
|
+
optLdm->startPosInBlock = UINT_MAX;
|
813
|
+
optLdm->endPosInBlock = UINT_MAX;
|
814
|
+
return;
|
815
|
+
}
|
816
|
+
/* Calculate appropriate bytes left in matchLength and litLength after adjusting
|
817
|
+
based on ldmSeqStore->posInSequence */
|
818
|
+
currSeq = optLdm->seqStore.seq[optLdm->seqStore.pos];
|
819
|
+
assert(optLdm->seqStore.posInSequence <= currSeq.litLength + currSeq.matchLength);
|
820
|
+
currBlockEndPos = currPosInBlock + blockBytesRemaining;
|
821
|
+
literalsBytesRemaining = (optLdm->seqStore.posInSequence < currSeq.litLength) ?
|
822
|
+
currSeq.litLength - (U32)optLdm->seqStore.posInSequence :
|
823
|
+
0;
|
824
|
+
matchBytesRemaining = (literalsBytesRemaining == 0) ?
|
825
|
+
currSeq.matchLength - ((U32)optLdm->seqStore.posInSequence - currSeq.litLength) :
|
826
|
+
currSeq.matchLength;
|
827
|
+
|
828
|
+
/* If there are more literal bytes than bytes remaining in block, no ldm is possible */
|
829
|
+
if (literalsBytesRemaining >= blockBytesRemaining) {
|
830
|
+
optLdm->startPosInBlock = UINT_MAX;
|
831
|
+
optLdm->endPosInBlock = UINT_MAX;
|
832
|
+
ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, blockBytesRemaining);
|
833
|
+
return;
|
834
|
+
}
|
835
|
+
|
836
|
+
/* Matches may be < MINMATCH by this process. In that case, we will reject them
|
837
|
+
when we are deciding whether or not to add the ldm */
|
838
|
+
optLdm->startPosInBlock = currPosInBlock + literalsBytesRemaining;
|
839
|
+
optLdm->endPosInBlock = optLdm->startPosInBlock + matchBytesRemaining;
|
840
|
+
optLdm->offset = currSeq.offset;
|
841
|
+
|
842
|
+
if (optLdm->endPosInBlock > currBlockEndPos) {
|
843
|
+
/* Match ends after the block ends, we can't use the whole match */
|
844
|
+
optLdm->endPosInBlock = currBlockEndPos;
|
845
|
+
ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, currBlockEndPos - currPosInBlock);
|
846
|
+
} else {
|
847
|
+
/* Consume nb of bytes equal to size of sequence left */
|
848
|
+
ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, literalsBytesRemaining + matchBytesRemaining);
|
849
|
+
}
|
850
|
+
}
|
851
|
+
|
852
|
+
/* ZSTD_optLdm_maybeAddMatch():
|
853
|
+
* Adds a match if it's long enough, based on it's 'matchStartPosInBlock'
|
854
|
+
* and 'matchEndPosInBlock', into 'matches'. Maintains the correct ordering of 'matches'
|
855
|
+
*/
|
856
|
+
static void ZSTD_optLdm_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches,
|
857
|
+
ZSTD_optLdm_t* optLdm, U32 currPosInBlock) {
|
858
|
+
U32 posDiff = currPosInBlock - optLdm->startPosInBlock;
|
859
|
+
/* Note: ZSTD_match_t actually contains offCode and matchLength (before subtracting MINMATCH) */
|
860
|
+
U32 candidateMatchLength = optLdm->endPosInBlock - optLdm->startPosInBlock - posDiff;
|
861
|
+
U32 candidateOffCode = optLdm->offset + ZSTD_REP_MOVE;
|
862
|
+
|
863
|
+
/* Ensure that current block position is not outside of the match */
|
864
|
+
if (currPosInBlock < optLdm->startPosInBlock
|
865
|
+
|| currPosInBlock >= optLdm->endPosInBlock
|
866
|
+
|| candidateMatchLength < MINMATCH) {
|
867
|
+
return;
|
868
|
+
}
|
869
|
+
|
870
|
+
if (*nbMatches == 0 || ((candidateMatchLength > matches[*nbMatches-1].len) && *nbMatches < ZSTD_OPT_NUM)) {
|
871
|
+
DEBUGLOG(6, "ZSTD_optLdm_maybeAddMatch(): Adding ldm candidate match (offCode: %u matchLength %u) at block position=%u",
|
872
|
+
candidateOffCode, candidateMatchLength, currPosInBlock);
|
873
|
+
matches[*nbMatches].len = candidateMatchLength;
|
874
|
+
matches[*nbMatches].off = candidateOffCode;
|
875
|
+
(*nbMatches)++;
|
876
|
+
}
|
877
|
+
}
|
878
|
+
|
879
|
+
/* ZSTD_optLdm_processMatchCandidate():
|
880
|
+
* Wrapper function to update ldm seq store and call ldm functions as necessary.
|
881
|
+
*/
|
882
|
+
static void ZSTD_optLdm_processMatchCandidate(ZSTD_optLdm_t* optLdm, ZSTD_match_t* matches, U32* nbMatches,
|
883
|
+
U32 currPosInBlock, U32 remainingBytes) {
|
884
|
+
if (optLdm->seqStore.size == 0 || optLdm->seqStore.pos >= optLdm->seqStore.size) {
|
885
|
+
return;
|
886
|
+
}
|
887
|
+
|
888
|
+
if (currPosInBlock >= optLdm->endPosInBlock) {
|
889
|
+
if (currPosInBlock > optLdm->endPosInBlock) {
|
890
|
+
/* The position at which ZSTD_optLdm_processMatchCandidate() is called is not necessarily
|
891
|
+
* at the end of a match from the ldm seq store, and will often be some bytes
|
892
|
+
* over beyond matchEndPosInBlock. As such, we need to correct for these "overshoots"
|
893
|
+
*/
|
894
|
+
U32 posOvershoot = currPosInBlock - optLdm->endPosInBlock;
|
895
|
+
ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, posOvershoot);
|
896
|
+
}
|
897
|
+
ZSTD_opt_getNextMatchAndUpdateSeqStore(optLdm, currPosInBlock, remainingBytes);
|
898
|
+
}
|
899
|
+
ZSTD_optLdm_maybeAddMatch(matches, nbMatches, optLdm, currPosInBlock);
|
900
|
+
}
|
767
901
|
|
768
902
|
/*-*******************************
|
769
903
|
* Optimal parser
|
@@ -817,6 +951,11 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
|
|
817
951
|
ZSTD_optimal_t* const opt = optStatePtr->priceTable;
|
818
952
|
ZSTD_match_t* const matches = optStatePtr->matchTable;
|
819
953
|
ZSTD_optimal_t lastSequence;
|
954
|
+
ZSTD_optLdm_t optLdm;
|
955
|
+
|
956
|
+
optLdm.seqStore = ms->ldmSeqStore ? *ms->ldmSeqStore : kNullRawSeqStore;
|
957
|
+
optLdm.endPosInBlock = optLdm.startPosInBlock = optLdm.offset = 0;
|
958
|
+
ZSTD_opt_getNextMatchAndUpdateSeqStore(&optLdm, (U32)(ip-istart), (U32)(iend-ip));
|
820
959
|
|
821
960
|
/* init */
|
822
961
|
DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u",
|
@@ -832,7 +971,9 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
|
|
832
971
|
/* find first match */
|
833
972
|
{ U32 const litlen = (U32)(ip - anchor);
|
834
973
|
U32 const ll0 = !litlen;
|
835
|
-
U32
|
974
|
+
U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, ip, iend, dictMode, rep, ll0, minMatch);
|
975
|
+
ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches,
|
976
|
+
(U32)(ip-istart), (U32)(iend - ip));
|
836
977
|
if (!nbMatches) { ip++; continue; }
|
837
978
|
|
838
979
|
/* initialize opt[0] */
|
@@ -925,9 +1066,9 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
|
|
925
1066
|
if (opt[cur].mlen != 0) {
|
926
1067
|
U32 const prev = cur - opt[cur].mlen;
|
927
1068
|
repcodes_t newReps = ZSTD_updateRep(opt[prev].rep, opt[cur].off, opt[cur].litlen==0);
|
928
|
-
|
1069
|
+
ZSTD_memcpy(opt[cur].rep, &newReps, sizeof(repcodes_t));
|
929
1070
|
} else {
|
930
|
-
|
1071
|
+
ZSTD_memcpy(opt[cur].rep, opt[cur - 1].rep, sizeof(repcodes_t));
|
931
1072
|
}
|
932
1073
|
|
933
1074
|
/* last match must start at a minimum distance of 8 from oend */
|
@@ -945,8 +1086,12 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
|
|
945
1086
|
U32 const litlen = (opt[cur].mlen == 0) ? opt[cur].litlen : 0;
|
946
1087
|
U32 const previousPrice = opt[cur].price;
|
947
1088
|
U32 const basePrice = previousPrice + ZSTD_litLengthPrice(0, optStatePtr, optLevel);
|
948
|
-
U32
|
1089
|
+
U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, inr, iend, dictMode, opt[cur].rep, ll0, minMatch);
|
949
1090
|
U32 matchNb;
|
1091
|
+
|
1092
|
+
ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches,
|
1093
|
+
(U32)(inr-istart), (U32)(iend-inr));
|
1094
|
+
|
950
1095
|
if (!nbMatches) {
|
951
1096
|
DEBUGLOG(7, "rPos:%u : no match found", cur);
|
952
1097
|
continue;
|
@@ -1010,9 +1155,9 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */
|
|
1010
1155
|
*/
|
1011
1156
|
if (lastSequence.mlen != 0) {
|
1012
1157
|
repcodes_t reps = ZSTD_updateRep(opt[cur].rep, lastSequence.off, lastSequence.litlen==0);
|
1013
|
-
|
1158
|
+
ZSTD_memcpy(rep, &reps, sizeof(reps));
|
1014
1159
|
} else {
|
1015
|
-
|
1160
|
+
ZSTD_memcpy(rep, opt[cur].rep, sizeof(repcodes_t));
|
1016
1161
|
}
|
1017
1162
|
|
1018
1163
|
{ U32 const storeEnd = cur + 1;
|
@@ -1110,7 +1255,7 @@ ZSTD_initStats_ultra(ZSTD_matchState_t* ms,
|
|
1110
1255
|
const void* src, size_t srcSize)
|
1111
1256
|
{
|
1112
1257
|
U32 tmpRep[ZSTD_REP_NUM]; /* updated rep codes will sink here */
|
1113
|
-
|
1258
|
+
ZSTD_memcpy(tmpRep, rep, sizeof(tmpRep));
|
1114
1259
|
|
1115
1260
|
DEBUGLOG(4, "ZSTD_initStats_ultra (srcSize=%zu)", srcSize);
|
1116
1261
|
assert(ms->opt.litLengthSum == 0); /* first block */
|
@@ -1143,7 +1288,7 @@ size_t ZSTD_compressBlock_btultra2(
|
|
1143
1288
|
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
|
1144
1289
|
const void* src, size_t srcSize)
|
1145
1290
|
{
|
1146
|
-
U32 const
|
1291
|
+
U32 const curr = (U32)((const BYTE*)src - ms->window.base);
|
1147
1292
|
DEBUGLOG(5, "ZSTD_compressBlock_btultra2 (srcSize=%zu)", srcSize);
|
1148
1293
|
|
1149
1294
|
/* 2-pass strategy:
|
@@ -1158,7 +1303,7 @@ size_t ZSTD_compressBlock_btultra2(
|
|
1158
1303
|
if ( (ms->opt.litLengthSum==0) /* first block */
|
1159
1304
|
&& (seqStore->sequences == seqStore->sequencesStart) /* no ldm */
|
1160
1305
|
&& (ms->window.dictLimit == ms->window.lowLimit) /* no dictionary */
|
1161
|
-
&& (
|
1306
|
+
&& (curr == ms->window.dictLimit) /* start of frame, nothing already loaded nor skipped */
|
1162
1307
|
&& (srcSize > ZSTD_PREDEF_THRESHOLD)
|
1163
1308
|
) {
|
1164
1309
|
ZSTD_initStats_ultra(ms, seqStore, rep, src, srcSize);
|