zstd-ruby 1.5.1.1 → 1.5.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/ext/zstdruby/libzstd/Makefile +2 -2
- data/ext/zstdruby/libzstd/common/pool.c +11 -6
- data/ext/zstdruby/libzstd/common/pool.h +2 -2
- data/ext/zstdruby/libzstd/common/portability_macros.h +6 -0
- data/ext/zstdruby/libzstd/common/zstd_internal.h +3 -4
- data/ext/zstdruby/libzstd/compress/zstd_compress.c +114 -96
- data/ext/zstdruby/libzstd/compress/zstd_compress_internal.h +72 -39
- data/ext/zstdruby/libzstd/compress/zstd_compress_sequences.c +10 -10
- data/ext/zstdruby/libzstd/compress/zstd_compress_superblock.c +1 -1
- data/ext/zstdruby/libzstd/compress/zstd_cwksp.h +38 -24
- data/ext/zstdruby/libzstd/compress/zstd_double_fast.c +10 -10
- data/ext/zstdruby/libzstd/compress/zstd_fast.c +11 -11
- data/ext/zstdruby/libzstd/compress/zstd_lazy.c +66 -62
- data/ext/zstdruby/libzstd/compress/zstd_ldm.c +5 -3
- data/ext/zstdruby/libzstd/compress/zstd_opt.c +66 -43
- data/ext/zstdruby/libzstd/compress/zstdmt_compress.c +17 -9
- data/ext/zstdruby/libzstd/compress/zstdmt_compress.h +4 -1
- data/ext/zstdruby/libzstd/decompress/huf_decompress.c +2 -2
- data/ext/zstdruby/libzstd/decompress/huf_decompress_amd64.S +17 -3
- data/ext/zstdruby/libzstd/dictBuilder/zdict.c +2 -2
- data/ext/zstdruby/libzstd/libzstd.mk +20 -2
- data/ext/zstdruby/libzstd/module.modulemap +25 -0
- data/ext/zstdruby/libzstd/zstd.h +1 -1
- data/lib/zstd-ruby/version.rb +1 -1
- metadata +7 -7
- data/ext/zstdruby/libzstd/modulemap/module.modulemap +0 -4
@@ -197,8 +197,8 @@ ZSTD_DUBT_findBetterDictMatch (
|
|
197
197
|
U32 matchIndex = dictMatchIndex + dictIndexDelta;
|
198
198
|
if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) {
|
199
199
|
DEBUGLOG(9, "ZSTD_DUBT_findBetterDictMatch(%u) : found better match length %u -> %u and offsetCode %u -> %u (dictMatchIndex %u, matchIndex %u)",
|
200
|
-
curr, (U32)bestLength, (U32)matchLength, (U32)*offsetPtr,
|
201
|
-
bestLength = matchLength, *offsetPtr =
|
200
|
+
curr, (U32)bestLength, (U32)matchLength, (U32)*offsetPtr, STORE_OFFSET(curr - matchIndex), dictMatchIndex, matchIndex);
|
201
|
+
bestLength = matchLength, *offsetPtr = STORE_OFFSET(curr - matchIndex);
|
202
202
|
}
|
203
203
|
if (ip+matchLength == iend) { /* reached end of input : ip[matchLength] is not valid, no way to know if it's larger or smaller than match */
|
204
204
|
break; /* drop, to guarantee consistency (miss a little bit of compression) */
|
@@ -218,7 +218,7 @@ ZSTD_DUBT_findBetterDictMatch (
|
|
218
218
|
}
|
219
219
|
|
220
220
|
if (bestLength >= MINMATCH) {
|
221
|
-
U32 const mIndex = curr - (
|
221
|
+
U32 const mIndex = curr - (U32)STORED_OFFSET(*offsetPtr); (void)mIndex;
|
222
222
|
DEBUGLOG(8, "ZSTD_DUBT_findBetterDictMatch(%u) : found match of length %u and offsetCode %u (pos %u)",
|
223
223
|
curr, (U32)bestLength, (U32)*offsetPtr, mIndex);
|
224
224
|
}
|
@@ -328,7 +328,7 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms,
|
|
328
328
|
if (matchLength > matchEndIdx - matchIndex)
|
329
329
|
matchEndIdx = matchIndex + (U32)matchLength;
|
330
330
|
if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) )
|
331
|
-
bestLength = matchLength, *offsetPtr =
|
331
|
+
bestLength = matchLength, *offsetPtr = STORE_OFFSET(curr - matchIndex);
|
332
332
|
if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */
|
333
333
|
if (dictMode == ZSTD_dictMatchState) {
|
334
334
|
nbCompares = 0; /* in addition to avoiding checking any
|
@@ -368,7 +368,7 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms,
|
|
368
368
|
assert(matchEndIdx > curr+8); /* ensure nextToUpdate is increased */
|
369
369
|
ms->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */
|
370
370
|
if (bestLength >= MINMATCH) {
|
371
|
-
U32 const mIndex = curr - (
|
371
|
+
U32 const mIndex = curr - (U32)STORED_OFFSET(*offsetPtr); (void)mIndex;
|
372
372
|
DEBUGLOG(8, "ZSTD_DUBT_findBestMatch(%u) : found match of length %u and offsetCode %u (pos %u)",
|
373
373
|
curr, (U32)bestLength, (U32)*offsetPtr, mIndex);
|
374
374
|
}
|
@@ -561,7 +561,7 @@ size_t ZSTD_dedicatedDictSearch_lazy_search(size_t* offsetPtr, size_t ml, U32 nb
|
|
561
561
|
/* save best solution */
|
562
562
|
if (currentMl > ml) {
|
563
563
|
ml = currentMl;
|
564
|
-
*offsetPtr = curr - (matchIndex + ddsIndexDelta)
|
564
|
+
*offsetPtr = STORE_OFFSET(curr - (matchIndex + ddsIndexDelta));
|
565
565
|
if (ip+currentMl == iLimit) {
|
566
566
|
/* best possible, avoids read overflow on next attempt */
|
567
567
|
return ml;
|
@@ -598,7 +598,7 @@ size_t ZSTD_dedicatedDictSearch_lazy_search(size_t* offsetPtr, size_t ml, U32 nb
|
|
598
598
|
/* save best solution */
|
599
599
|
if (currentMl > ml) {
|
600
600
|
ml = currentMl;
|
601
|
-
*offsetPtr = curr - (matchIndex + ddsIndexDelta)
|
601
|
+
*offsetPtr = STORE_OFFSET(curr - (matchIndex + ddsIndexDelta));
|
602
602
|
if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */
|
603
603
|
}
|
604
604
|
}
|
@@ -703,7 +703,7 @@ size_t ZSTD_HcFindBestMatch(
|
|
703
703
|
/* save best solution */
|
704
704
|
if (currentMl > ml) {
|
705
705
|
ml = currentMl;
|
706
|
-
*offsetPtr = curr - matchIndex
|
706
|
+
*offsetPtr = STORE_OFFSET(curr - matchIndex);
|
707
707
|
if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */
|
708
708
|
}
|
709
709
|
|
@@ -738,7 +738,8 @@ size_t ZSTD_HcFindBestMatch(
|
|
738
738
|
/* save best solution */
|
739
739
|
if (currentMl > ml) {
|
740
740
|
ml = currentMl;
|
741
|
-
|
741
|
+
assert(curr > matchIndex + dmsIndexDelta);
|
742
|
+
*offsetPtr = STORE_OFFSET(curr - (matchIndex + dmsIndexDelta));
|
742
743
|
if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */
|
743
744
|
}
|
744
745
|
|
@@ -1244,7 +1245,7 @@ size_t ZSTD_RowFindBestMatch(
|
|
1244
1245
|
/* Save best solution */
|
1245
1246
|
if (currentMl > ml) {
|
1246
1247
|
ml = currentMl;
|
1247
|
-
*offsetPtr = curr - matchIndex
|
1248
|
+
*offsetPtr = STORE_OFFSET(curr - matchIndex);
|
1248
1249
|
if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */
|
1249
1250
|
}
|
1250
1251
|
}
|
@@ -1292,7 +1293,8 @@ size_t ZSTD_RowFindBestMatch(
|
|
1292
1293
|
|
1293
1294
|
if (currentMl > ml) {
|
1294
1295
|
ml = currentMl;
|
1295
|
-
|
1296
|
+
assert(curr > matchIndex + dmsIndexDelta);
|
1297
|
+
*offsetPtr = STORE_OFFSET(curr - (matchIndex + dmsIndexDelta));
|
1296
1298
|
if (ip+currentMl == iLimit) break;
|
1297
1299
|
}
|
1298
1300
|
}
|
@@ -1537,8 +1539,9 @@ ZSTD_compressBlock_lazy_generic(
|
|
1537
1539
|
#endif
|
1538
1540
|
while (ip < ilimit) {
|
1539
1541
|
size_t matchLength=0;
|
1540
|
-
size_t
|
1542
|
+
size_t offcode=STORE_REPCODE_1;
|
1541
1543
|
const BYTE* start=ip+1;
|
1544
|
+
DEBUGLOG(7, "search baseline (depth 0)");
|
1542
1545
|
|
1543
1546
|
/* check repCode */
|
1544
1547
|
if (isDxS) {
|
@@ -1564,7 +1567,7 @@ ZSTD_compressBlock_lazy_generic(
|
|
1564
1567
|
{ size_t offsetFound = 999999999;
|
1565
1568
|
size_t const ml2 = searchMax(ms, ip, iend, &offsetFound);
|
1566
1569
|
if (ml2 > matchLength)
|
1567
|
-
matchLength = ml2, start = ip,
|
1570
|
+
matchLength = ml2, start = ip, offcode=offsetFound;
|
1568
1571
|
}
|
1569
1572
|
|
1570
1573
|
if (matchLength < 4) {
|
@@ -1575,14 +1578,15 @@ ZSTD_compressBlock_lazy_generic(
|
|
1575
1578
|
/* let's try to find a better solution */
|
1576
1579
|
if (depth>=1)
|
1577
1580
|
while (ip<ilimit) {
|
1581
|
+
DEBUGLOG(7, "search depth 1");
|
1578
1582
|
ip ++;
|
1579
1583
|
if ( (dictMode == ZSTD_noDict)
|
1580
|
-
&& (
|
1584
|
+
&& (offcode) && ((offset_1>0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) {
|
1581
1585
|
size_t const mlRep = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4;
|
1582
1586
|
int const gain2 = (int)(mlRep * 3);
|
1583
|
-
int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)
|
1587
|
+
int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 1);
|
1584
1588
|
if ((mlRep >= 4) && (gain2 > gain1))
|
1585
|
-
matchLength = mlRep,
|
1589
|
+
matchLength = mlRep, offcode = STORE_REPCODE_1, start = ip;
|
1586
1590
|
}
|
1587
1591
|
if (isDxS) {
|
1588
1592
|
const U32 repIndex = (U32)(ip - base) - offset_1;
|
@@ -1594,30 +1598,31 @@ ZSTD_compressBlock_lazy_generic(
|
|
1594
1598
|
const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
|
1595
1599
|
size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
|
1596
1600
|
int const gain2 = (int)(mlRep * 3);
|
1597
|
-
int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)
|
1601
|
+
int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 1);
|
1598
1602
|
if ((mlRep >= 4) && (gain2 > gain1))
|
1599
|
-
matchLength = mlRep,
|
1603
|
+
matchLength = mlRep, offcode = STORE_REPCODE_1, start = ip;
|
1600
1604
|
}
|
1601
1605
|
}
|
1602
1606
|
{ size_t offset2=999999999;
|
1603
1607
|
size_t const ml2 = searchMax(ms, ip, iend, &offset2);
|
1604
|
-
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2
|
1605
|
-
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)
|
1608
|
+
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offset2))); /* raw approx */
|
1609
|
+
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 4);
|
1606
1610
|
if ((ml2 >= 4) && (gain2 > gain1)) {
|
1607
|
-
matchLength = ml2,
|
1611
|
+
matchLength = ml2, offcode = offset2, start = ip;
|
1608
1612
|
continue; /* search a better one */
|
1609
1613
|
} }
|
1610
1614
|
|
1611
1615
|
/* let's find an even better one */
|
1612
1616
|
if ((depth==2) && (ip<ilimit)) {
|
1617
|
+
DEBUGLOG(7, "search depth 2");
|
1613
1618
|
ip ++;
|
1614
1619
|
if ( (dictMode == ZSTD_noDict)
|
1615
|
-
&& (
|
1620
|
+
&& (offcode) && ((offset_1>0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) {
|
1616
1621
|
size_t const mlRep = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4;
|
1617
1622
|
int const gain2 = (int)(mlRep * 4);
|
1618
|
-
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)
|
1623
|
+
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 1);
|
1619
1624
|
if ((mlRep >= 4) && (gain2 > gain1))
|
1620
|
-
matchLength = mlRep,
|
1625
|
+
matchLength = mlRep, offcode = STORE_REPCODE_1, start = ip;
|
1621
1626
|
}
|
1622
1627
|
if (isDxS) {
|
1623
1628
|
const U32 repIndex = (U32)(ip - base) - offset_1;
|
@@ -1629,46 +1634,45 @@ ZSTD_compressBlock_lazy_generic(
|
|
1629
1634
|
const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend;
|
1630
1635
|
size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4;
|
1631
1636
|
int const gain2 = (int)(mlRep * 4);
|
1632
|
-
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)
|
1637
|
+
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 1);
|
1633
1638
|
if ((mlRep >= 4) && (gain2 > gain1))
|
1634
|
-
matchLength = mlRep,
|
1639
|
+
matchLength = mlRep, offcode = STORE_REPCODE_1, start = ip;
|
1635
1640
|
}
|
1636
1641
|
}
|
1637
1642
|
{ size_t offset2=999999999;
|
1638
1643
|
size_t const ml2 = searchMax(ms, ip, iend, &offset2);
|
1639
|
-
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2
|
1640
|
-
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)
|
1644
|
+
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offset2))); /* raw approx */
|
1645
|
+
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 7);
|
1641
1646
|
if ((ml2 >= 4) && (gain2 > gain1)) {
|
1642
|
-
matchLength = ml2,
|
1647
|
+
matchLength = ml2, offcode = offset2, start = ip;
|
1643
1648
|
continue;
|
1644
1649
|
} } }
|
1645
1650
|
break; /* nothing found : store previous solution */
|
1646
1651
|
}
|
1647
1652
|
|
1648
1653
|
/* NOTE:
|
1649
|
-
* start[-
|
1650
|
-
*
|
1651
|
-
* overflows the pointer, which is undefined behavior.
|
1654
|
+
* Pay attention that `start[-value]` can lead to strange undefined behavior
|
1655
|
+
* notably if `value` is unsigned, resulting in a large positive `-value`.
|
1652
1656
|
*/
|
1653
1657
|
/* catch up */
|
1654
|
-
if (
|
1658
|
+
if (STORED_IS_OFFSET(offcode)) {
|
1655
1659
|
if (dictMode == ZSTD_noDict) {
|
1656
|
-
while ( ((start > anchor) & (start - (
|
1657
|
-
&& (start[-1] == (start-(
|
1660
|
+
while ( ((start > anchor) & (start - STORED_OFFSET(offcode) > prefixLowest))
|
1661
|
+
&& (start[-1] == (start-STORED_OFFSET(offcode))[-1]) ) /* only search for offset within prefix */
|
1658
1662
|
{ start--; matchLength++; }
|
1659
1663
|
}
|
1660
1664
|
if (isDxS) {
|
1661
|
-
U32 const matchIndex = (U32)((size_t)(start-base) - (
|
1665
|
+
U32 const matchIndex = (U32)((size_t)(start-base) - STORED_OFFSET(offcode));
|
1662
1666
|
const BYTE* match = (matchIndex < prefixLowestIndex) ? dictBase + matchIndex - dictIndexDelta : base + matchIndex;
|
1663
1667
|
const BYTE* const mStart = (matchIndex < prefixLowestIndex) ? dictLowest : prefixLowest;
|
1664
1668
|
while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */
|
1665
1669
|
}
|
1666
|
-
offset_2 = offset_1; offset_1 = (U32)(
|
1670
|
+
offset_2 = offset_1; offset_1 = (U32)STORED_OFFSET(offcode);
|
1667
1671
|
}
|
1668
1672
|
/* store sequence */
|
1669
1673
|
_storeSequence:
|
1670
1674
|
{ size_t const litLength = (size_t)(start - anchor);
|
1671
|
-
ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)
|
1675
|
+
ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offcode, matchLength);
|
1672
1676
|
anchor = ip = start + matchLength;
|
1673
1677
|
}
|
1674
1678
|
|
@@ -1684,8 +1688,8 @@ _storeSequence:
|
|
1684
1688
|
&& (MEM_read32(repMatch) == MEM_read32(ip)) ) {
|
1685
1689
|
const BYTE* const repEnd2 = repIndex < prefixLowestIndex ? dictEnd : iend;
|
1686
1690
|
matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd2, prefixLowest) + 4;
|
1687
|
-
|
1688
|
-
ZSTD_storeSeq(seqStore, 0, anchor, iend,
|
1691
|
+
offcode = offset_2; offset_2 = offset_1; offset_1 = (U32)offcode; /* swap offset_2 <=> offset_1 */
|
1692
|
+
ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, matchLength);
|
1689
1693
|
ip += matchLength;
|
1690
1694
|
anchor = ip;
|
1691
1695
|
continue;
|
@@ -1699,8 +1703,8 @@ _storeSequence:
|
|
1699
1703
|
&& (MEM_read32(ip) == MEM_read32(ip - offset_2)) ) {
|
1700
1704
|
/* store sequence */
|
1701
1705
|
matchLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4;
|
1702
|
-
|
1703
|
-
ZSTD_storeSeq(seqStore, 0, anchor, iend,
|
1706
|
+
offcode = offset_2; offset_2 = offset_1; offset_1 = (U32)offcode; /* swap repcodes */
|
1707
|
+
ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, matchLength);
|
1704
1708
|
ip += matchLength;
|
1705
1709
|
anchor = ip;
|
1706
1710
|
continue; /* faster when present ... (?) */
|
@@ -1901,7 +1905,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
|
|
1901
1905
|
#endif
|
1902
1906
|
while (ip < ilimit) {
|
1903
1907
|
size_t matchLength=0;
|
1904
|
-
size_t
|
1908
|
+
size_t offcode=STORE_REPCODE_1;
|
1905
1909
|
const BYTE* start=ip+1;
|
1906
1910
|
U32 curr = (U32)(ip-base);
|
1907
1911
|
|
@@ -1923,7 +1927,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
|
|
1923
1927
|
{ size_t offsetFound = 999999999;
|
1924
1928
|
size_t const ml2 = searchMax(ms, ip, iend, &offsetFound);
|
1925
1929
|
if (ml2 > matchLength)
|
1926
|
-
matchLength = ml2, start = ip,
|
1930
|
+
matchLength = ml2, start = ip, offcode=offsetFound;
|
1927
1931
|
}
|
1928
1932
|
|
1929
1933
|
if (matchLength < 4) {
|
@@ -1937,7 +1941,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
|
|
1937
1941
|
ip ++;
|
1938
1942
|
curr++;
|
1939
1943
|
/* check repCode */
|
1940
|
-
if (
|
1944
|
+
if (offcode) {
|
1941
1945
|
const U32 windowLow = ZSTD_getLowestMatchIndex(ms, curr, windowLog);
|
1942
1946
|
const U32 repIndex = (U32)(curr - offset_1);
|
1943
1947
|
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
|
@@ -1949,18 +1953,18 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
|
|
1949
1953
|
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
|
1950
1954
|
size_t const repLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4;
|
1951
1955
|
int const gain2 = (int)(repLength * 3);
|
1952
|
-
int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)
|
1956
|
+
int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 1);
|
1953
1957
|
if ((repLength >= 4) && (gain2 > gain1))
|
1954
|
-
matchLength = repLength,
|
1958
|
+
matchLength = repLength, offcode = STORE_REPCODE_1, start = ip;
|
1955
1959
|
} }
|
1956
1960
|
|
1957
1961
|
/* search match, depth 1 */
|
1958
1962
|
{ size_t offset2=999999999;
|
1959
1963
|
size_t const ml2 = searchMax(ms, ip, iend, &offset2);
|
1960
|
-
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2
|
1961
|
-
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)
|
1964
|
+
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offset2))); /* raw approx */
|
1965
|
+
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 4);
|
1962
1966
|
if ((ml2 >= 4) && (gain2 > gain1)) {
|
1963
|
-
matchLength = ml2,
|
1967
|
+
matchLength = ml2, offcode = offset2, start = ip;
|
1964
1968
|
continue; /* search a better one */
|
1965
1969
|
} }
|
1966
1970
|
|
@@ -1969,7 +1973,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
|
|
1969
1973
|
ip ++;
|
1970
1974
|
curr++;
|
1971
1975
|
/* check repCode */
|
1972
|
-
if (
|
1976
|
+
if (offcode) {
|
1973
1977
|
const U32 windowLow = ZSTD_getLowestMatchIndex(ms, curr, windowLog);
|
1974
1978
|
const U32 repIndex = (U32)(curr - offset_1);
|
1975
1979
|
const BYTE* const repBase = repIndex < dictLimit ? dictBase : base;
|
@@ -1981,36 +1985,36 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
|
|
1981
1985
|
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
|
1982
1986
|
size_t const repLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4;
|
1983
1987
|
int const gain2 = (int)(repLength * 4);
|
1984
|
-
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)
|
1988
|
+
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 1);
|
1985
1989
|
if ((repLength >= 4) && (gain2 > gain1))
|
1986
|
-
matchLength = repLength,
|
1990
|
+
matchLength = repLength, offcode = STORE_REPCODE_1, start = ip;
|
1987
1991
|
} }
|
1988
1992
|
|
1989
1993
|
/* search match, depth 2 */
|
1990
1994
|
{ size_t offset2=999999999;
|
1991
1995
|
size_t const ml2 = searchMax(ms, ip, iend, &offset2);
|
1992
|
-
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2
|
1993
|
-
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)
|
1996
|
+
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offset2))); /* raw approx */
|
1997
|
+
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 7);
|
1994
1998
|
if ((ml2 >= 4) && (gain2 > gain1)) {
|
1995
|
-
matchLength = ml2,
|
1999
|
+
matchLength = ml2, offcode = offset2, start = ip;
|
1996
2000
|
continue;
|
1997
2001
|
} } }
|
1998
2002
|
break; /* nothing found : store previous solution */
|
1999
2003
|
}
|
2000
2004
|
|
2001
2005
|
/* catch up */
|
2002
|
-
if (
|
2003
|
-
U32 const matchIndex = (U32)((size_t)(start-base) - (
|
2006
|
+
if (STORED_IS_OFFSET(offcode)) {
|
2007
|
+
U32 const matchIndex = (U32)((size_t)(start-base) - STORED_OFFSET(offcode));
|
2004
2008
|
const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex;
|
2005
2009
|
const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart;
|
2006
2010
|
while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */
|
2007
|
-
offset_2 = offset_1; offset_1 = (U32)(
|
2011
|
+
offset_2 = offset_1; offset_1 = (U32)STORED_OFFSET(offcode);
|
2008
2012
|
}
|
2009
2013
|
|
2010
2014
|
/* store sequence */
|
2011
2015
|
_storeSequence:
|
2012
2016
|
{ size_t const litLength = (size_t)(start - anchor);
|
2013
|
-
ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)
|
2017
|
+
ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offcode, matchLength);
|
2014
2018
|
anchor = ip = start + matchLength;
|
2015
2019
|
}
|
2016
2020
|
|
@@ -2027,8 +2031,8 @@ _storeSequence:
|
|
2027
2031
|
/* repcode detected we should take it */
|
2028
2032
|
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
|
2029
2033
|
matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4;
|
2030
|
-
|
2031
|
-
ZSTD_storeSeq(seqStore, 0, anchor, iend,
|
2034
|
+
offcode = offset_2; offset_2 = offset_1; offset_1 = (U32)offcode; /* swap offset history */
|
2035
|
+
ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, matchLength);
|
2032
2036
|
ip += matchLength;
|
2033
2037
|
anchor = ip;
|
2034
2038
|
continue; /* faster when present ... (?) */
|
@@ -579,7 +579,9 @@ size_t ZSTD_ldm_generateSequences(
|
|
579
579
|
return 0;
|
580
580
|
}
|
581
581
|
|
582
|
-
void
|
582
|
+
void
|
583
|
+
ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 const minMatch)
|
584
|
+
{
|
583
585
|
while (srcSize > 0 && rawSeqStore->pos < rawSeqStore->size) {
|
584
586
|
rawSeq* seq = rawSeqStore->seq + rawSeqStore->pos;
|
585
587
|
if (srcSize <= seq->litLength) {
|
@@ -709,8 +711,8 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
|
|
709
711
|
rep[0] = sequence.offset;
|
710
712
|
/* Store the sequence */
|
711
713
|
ZSTD_storeSeq(seqStore, newLitLength, ip - newLitLength, iend,
|
712
|
-
sequence.offset
|
713
|
-
sequence.matchLength
|
714
|
+
STORE_OFFSET(sequence.offset),
|
715
|
+
sequence.matchLength);
|
714
716
|
ip += sequence.matchLength;
|
715
717
|
}
|
716
718
|
}
|